iSulad: add isulad-shim to support oci runtime

Signed-off-by: Nleizhongkai <leizhongkai@huawei.com>

iSulad.spec

@@ -68,6 +68,7 @@ install -m 0640 ./conf/isulad.pc              %{buildroot}/%{_libdir}/pkgconfig/
 
 install -d $RPM_BUILD_ROOT/%{_bindir}
 install -m 0755 ./src/isula                  %{buildroot}/%{_bindir}/isula
+install -m 0755 ./src/isulad-shim            %{buildroot}/%{_bindir}/isulad-shim
 install -m 0755 ./src/isulad                  %{buildroot}/%{_bindir}/isulad
 
 install -d $RPM_BUILD_ROOT/%{_includedir}/isulad

src/CMakeLists.txt

@@ -143,6 +143,18 @@ target_include_directories(isula PUBLIC ${ISULA_INCS} ${SHARED_INCS})
 target_link_libraries(isula libisula -lpthread)
 # ------ build isula finish -------
 
+# ------ build isulad-shim -------
+add_executable(isulad-shim
+    ${ISULAD_SHIM_SRCS}
+    ${CMAKE_BINARY_DIR}/json/shim_client_process_state.c
+    ${CMAKE_BINARY_DIR}/json/json_common.c
+    ${commonjsonsrcs}
+    )
+target_include_directories(isulad-shim PUBLIC ${ISULAD_SHIM_INCS} ${SHARED_INCS})
+target_link_libraries(isulad-shim ${LIBYAJL_LIBRARY} -lpthread)
+
+# ------ build isula-shim finish -------
+
 # ------ build isulad -------
 add_subdirectory(services)
 add_subdirectory(image)
@@ -237,5 +249,7 @@ install(TARGETS libisula
   LIBRARY DESTINATION ${LIB_INSTALL_DIR_DEFAULT} PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE)
 install(TARGETS isula
 	RUNTIME DESTINATION bin PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE)
+install(TARGETS isulad-shim
+	RUNTIME DESTINATION bin PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE)
 install(TARGETS isulad
 	RUNTIME DESTINATION bin PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE)

src/cmd/CMakeLists.txt

@@ -8,3 +8,7 @@ set(ISULA_INCS ${CMAKE_CURRENT_SOURCE_DIR} ${CMD_ISULA_INCS} PARENT_SCOPE)
 add_subdirectory(isulad)
 set(ISULAD_SRCS ${comm_srcs} ${CMD_ISULAD_SRCS} PARENT_SCOPE)
 set(ISULAD_INCS ${CMAKE_CURRENT_SOURCE_DIR} ${CMD_ISULAD_INCS} PARENT_SCOPE)
+
+add_subdirectory(isulad-shim)
+set(ISULAD_SHIM_SRCS  ${CMD_ISULAD_SHIM_SRCS} PARENT_SCOPE)
+set(ISULAD_SHIM_INCS ${CMAKE_CURRENT_SOURCE_DIR} ${CMD_ISULAD_SHIM_INCS} PARENT_SCOPE)

src/cmd/isulad-shim/CMakeLists.txt

+# get current directory sources files
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} isulad_shim_srcs)
+
+set(CMD_ISULAD_SHIM_SRCS
+    ${isulad_shim_srcs}
+    PARENT_SCOPE
+    )
+
+set(CMD_ISULAD_SHIM_INCS
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    PARENT_SCOPE
+    )

src/cmd/isulad-shim/common.c

+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
+ * iSulad licensed under the Mulan PSL v1.
+ * You can use this software according to the terms and conditions of the Mulan PSL v1.
+ * You may obtain a copy of Mulan PSL v1 at:
+ *     http://license.coscl.org.cn/MulanPSL
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v1 for more details.
+ * Author: leizhongkai
+ * Create: 2020-1-21
+ * Description: common functions of isulad-shim
+ ******************************************************************************/
+
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <linux/limits.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <signal.h>
+#include <sys/wait.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <stdbool.h>
+#include <stdarg.h>
+
+#include "common.h"
+
+extern int g_log_fd;
+
+int set_fd_no_inherited(int fd)
+{
+    int ret = SHIM_ERR;
+    int flag = -1;
+
+    flag = fcntl(fd, F_GETFD, 0);
+    if (flag < 0) {
+        return SHIM_ERR;
+    }
+
+    ret = fcntl(fd, F_SETFD, flag | FD_CLOEXEC);
+    if (ret != 0) {
+        return SHIM_ERR;
+    }
+
+    return SHIM_OK;
+}
+
+ssize_t read_nointr(int fd, void *buf, size_t count)
+{
+    ssize_t nret;
+
+    if (buf == NULL) {
+        return -1;
+    }
+
+    for (;;) {
+        nret = read(fd, buf, count);
+        if (nret < 0 && (errno == EINTR || errno == EAGAIN)) {
+            continue;
+        } else {
+            break;
+        }
+    }
+
+    return nret;
+}
+
+ssize_t write_nointr(int fd, const void *buf, size_t count)
+{
+    ssize_t nret;
+
+    if (buf == NULL) {
+        return -1;
+    }
+
+    for (;;) {
+        nret = write(fd, buf, count);
+        if (nret < 0 && (errno == EINTR || errno == EAGAIN)) {
+            continue;
+        } else {
+            break;
+        }
+    }
+    return nret;
+}
+
+bool file_exists(const char *f)
+{
+    struct stat buf;
+    int nret;
+
+    if (f == NULL) {
+        return false;
+    }
+
+    nret = stat(f, &buf);
+    if (nret < 0) {
+        return false;
+    }
+    return true;
+}
+
+int cmd_combined_output(const char *binary, const char *params[], void *output, int *output_len)
+{
+    int ret = SHIM_ERR;
+    int exec_fd[2] = { -1, -1 };
+    int stdio[2] = { -1, -1 };
+    pid_t pid = 0;
+    char exec_buff[BUFSIZ + 1] = { 0 };
+    ssize_t nread;
+
+    if (pipe2(exec_fd, O_CLOEXEC) != 0) {
+        return SHIM_ERR;
+    }
+
+    if (pipe2(stdio, O_CLOEXEC) != 0) {
+        return SHIM_ERR;
+    }
+
+    pid = fork();
+    if (pid == (pid_t) - 1) {
+        return SHIM_ERR;
+    }
+
+    // child
+    if (pid == (pid_t)0) {
+        close(exec_fd[0]);
+        close(stdio[0]);
+        dup2(stdio[1], 1);
+        dup2(stdio[1], 2);
+        execvp(binary, (char * const *)params);
+        (void)dprintf(exec_fd[1], "fork/exec error: %s", strerror(errno));
+    }
+
+    // parent
+    close(exec_fd[1]);
+    close(stdio[1]);
+    nread = read_nointr(exec_fd[0], exec_buff, sizeof(exec_buff));
+    if (nread > 0) {
+        ret = SHIM_ERR;
+        goto out;
+    }
+    *output_len = read_nointr(stdio[0], output, 8191);
+
+    close(stdio[0]);
+    close(exec_fd[0]);
+    int status = 0;
+    wait(&status);
+    ret = SHIM_OK;
+out:
+    if (ret != SHIM_OK && pid != 0) {
+        kill(pid, 9);
+    }
+
+    return ret;
+}
+
+int generate_random_str(char *id, size_t len)
+{
+    int fd = -1;
+    int num = 0;
+    size_t i;
+    const int m = 256;
+
+    len = len / 2;
+    fd = open("/dev/urandom", O_RDONLY);
+    if (fd == -1) {
+        return SHIM_ERR;
+    }
+    for (i = 0; i < len; i++) {
+        int nret;
+        if (read(fd, &num, sizeof(int)) < 0) {
+            close(fd);
+            return SHIM_ERR;
+        }
+        unsigned char rs = (unsigned char)(num % m);
+        nret = snprintf((id + i * 2), ((len - i) * 2 + 1), "%02x", (unsigned int)rs);
+        if (nret < 0) {
+            close(fd);
+            return SHIM_ERR;
+        }
+    }
+    close(fd);
+    id[i * 2] = '\0';
+
+    return SHIM_OK;
+}
+
+void write_message(int fd, const char *level, const char *fmt, ...)
+{
+#define MAX_MSG_JSON_TEMPLATE 32
+#define MAX_MESSAGE_CONTENT_LEN 128
+#define MAX_MESSAGE_LEN (MAX_MSG_JSON_TEMPLATE + MAX_MESSAGE_CONTENT_LEN)
+    if (fd < 0) {
+        return;
+    }
+
+    char buf[MAX_MESSAGE_CONTENT_LEN] = { 0 };
+    char msg[MAX_MESSAGE_LEN] = { 0 };
+    int nwrite = -1;
+
+    va_list arg_list;
+    va_start(arg_list, fmt);
+    vsnprintf(buf, MAX_MESSAGE_CONTENT_LEN, fmt, arg_list);
+    va_end(arg_list);
+
+    snprintf(msg, MAX_MESSAGE_LEN - 1, "{\"level\": \"%s\", \"msg\": \"%s\"}\n", level, buf);
+    nwrite = write(fd, msg, strlen(msg));
+    if (nwrite != strlen(msg)) {
+        return;
+    }
+
+    return;
+}
+
+/* note: This function can only read small text file. */
+char *read_text_file(const char *path)
+{
+    char *buf = NULL;
+    long len = 0;
+    size_t readlen = 0;
+    FILE *filp = NULL;
+    const long max_size = 10 * 1024 * 1024; /* 10M */
+
+    if (path == NULL) {
+        return NULL;
+    }
+
+    filp = fopen(path, "r");
+    if (filp == NULL) {
+        goto err_out;
+    }
+    if (fseek(filp, 0, SEEK_END)) {
+        goto err_out;
+    }
+
+    len = ftell(filp);
+    if (len > max_size) {
+        goto err_out;
+    }
+    if (fseek(filp, 0, SEEK_SET)) {
+        goto err_out;
+    }
+
+    buf = (char *)calloc(1, (size_t)(len + 1));
+    if (buf == NULL) {
+        goto err_out;
+    }
+
+    readlen = fread(buf, 1, (size_t)len, filp);
+    if (((readlen < (size_t)len) && (!feof(filp))) || (readlen > (size_t)len)) {
+        if (buf != NULL) {
+            free(buf);
+            buf = NULL;
+        }
+        goto err_out;
+    }
+
+    buf[(size_t)len] = 0;
+
+err_out:
+
+    if (filp != NULL) {
+        fclose(filp);
+    }
+
+    return buf;
+}
+
+void close_fd(int *pfd)
+{
+    if (pfd != NULL && *pfd != -1) {
+        close(*pfd);
+        *pfd = -1;
+    }
+}
+
+int open_no_inherit(const char *path, int flag, mode_t mode)
+{
+    int fd = -1;
+    int ret = SHIM_ERR;
+
+    fd = open(path, flag, mode);
+    if (fd < 0) {
+        return -1;
+    }
+
+    ret = set_fd_no_inherited(fd);
+    if (ret != SHIM_OK) {
+        close(fd);
+        return -1;
+    }
+
+    return fd;
+}
+

src/cmd/isulad-shim/common.h

+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
+ * iSulad licensed under the Mulan PSL v1.
+ * You can use this software according to the terms and conditions of the Mulan PSL v1.
+ * You may obtain a copy of Mulan PSL v1 at:
+ *     http://license.coscl.org.cn/MulanPSL
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v1 for more details.
+ * Author: leizhongkai
+ * Create: 2020-1-20
+ * Description: common definition of isulad-shim
+ ******************************************************************************/
+
+#ifndef __COMMON_H_
+#define __COMMON_H_
+
+#include <stdbool.h>
+
+// error code
+#define SHIM_ERR_BASE (-10000)
+#define SHIM_SYS_ERR(err) (SHIM_ERR_BASE-err)
+#define SHIM_OK    0
+#define SHIM_ERR  -1
+#define SHIM_ERR_WAIT -2
+#define SHIM_ERR_NOT_REQUIRED -3
+
+#define INFO_MSG "info"
+#define WARN_MSG "warn"
+#define ERR_MSG "error"
+
+#define DEFAULT_TIMEOUT   120 // sec
+#define CONTAINER_ID_LEN  64
+#define MAX_RT_NAME_LEN   64
+#define MAX_CONSOLE_SOCK_LEN 32
+
+#define MAX_RUNTIME_ARGS 20
+
+#define SHIM_BINARY "isulad-shim"
+#define SHIM_LOG_NAME "shim-log.json"
+
+#define CONTAINER_ACTION_REBOOT 129
+#define CONTAINER_ACTION_SHUTDOWN 130
+
+ssize_t read_nointr(int fd, void *buf, size_t count);
+ssize_t write_nointr(int fd, const void *buf, size_t count);
+
+char *read_text_file(const char *path);
+
+bool file_exists(const char *f);
+
+int cmd_combined_output(const char *binary, const char *params[], void *output, int *output_len);
+
+void write_message(int fd, const char *level, const char *fmt, ...);
+
+int generate_random_str(char *id, size_t len);
+
+void close_fd(int *pfd);
+
+int open_no_inherit(const char *path, int flag, mode_t mode);
+
+#endif
+

src/cmd/isulad-shim/main.c

+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
+ * iSulad licensed under the Mulan PSL v1.
+ * You can use this software according to the terms and conditions of the Mulan PSL v1.
+ * You may obtain a copy of Mulan PSL v1 at:
+ *     http://license.coscl.org.cn/MulanPSL
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v1 for more details.
+ * Author: leizhongkai
+ * Create: 2020-1-20
+ * Description: main process of isulad-shim
+ ******************************************************************************/
+
+#include <stdio.h>
+#include <unistd.h>
+#include <signal.h>
+#include <limits.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/prctl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "common.h"
+#include "process.h"
+
+int g_log_fd = -1;
+
+void signal_routine(int sig)
+{
+    switch (sig) {
+        case SIGALRM:
+            write_message(g_log_fd, ERR_MSG, "runtime timeout");
+            exit(1);
+        default:
+            break;
+    }
+}
+
+static void set_timeout_exit(unsigned int timeout)
+{
+    signal(SIGALRM, signal_routine);
+    (void)alarm(timeout);
+}
+
+static void released_timeout_exit()
+{
+    (void)alarm(0);
+    signal(SIGALRM, SIG_IGN);
+}
+
+static int set_subreaper()
+{
+    int ret = SHIM_ERR;
+    ret = prctl(PR_SET_CHILD_SUBREAPER, 1);
+    if (ret != SHIM_OK) {
+        return SHIM_SYS_ERR(errno);
+    }
+
+    return SHIM_OK;
+}
+
+static int parse_args(int argc, char **argv, char **cid, char **bundle, char **rt_name, char **log_level)
+{
+    if (argc < 4) {
+        return SHIM_ERR;
+    }
+
+    *cid = strdup(argv[1]);
+    *bundle = strdup(argv[2]);
+    *rt_name = strdup(argv[3]);
+    if (*cid == NULL || *bundle == NULL || rt_name == NULL) {
+        return SHIM_ERR;
+    }
+
+    if (argc > 4) {
+        *log_level = strdup(argv[4]);
+        if (*log_level == NULL) {
+            return SHIM_ERR;
+        }
+    }
+
+    return SHIM_OK;
+}
+
+
+int main(int argc, char **argv)
+{
+    char *container_id = NULL;
+    char *bundle = NULL;
+    char *rt_name = NULL;
+    char *log_level = NULL;
+    int ret = SHIM_ERR;
+    int efd = -1;
+    process_t *p = NULL;
+
+    g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640);
+    if (g_log_fd < 0) {
+        _exit(EXIT_FAILURE);
+    }
+
+    set_timeout_exit(DEFAULT_TIMEOUT);
+
+    ret = set_subreaper();
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "set subreaper failed:%d", ret);
+        exit(EXIT_FAILURE);
+    }
+
+    ret = parse_args(argc, argv, &container_id, &bundle, &rt_name, &log_level);
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "parse args failed:%d", ret);
+        exit(EXIT_FAILURE);
+    }
+
+    p = new_process(container_id, bundle, rt_name);
+    if (p == NULL) {
+        write_message(g_log_fd, ERR_MSG, "new process failed");
+        exit(EXIT_FAILURE);
+    }
+
+    // open exit pipe
+    if (!p->state->exec) {
+        if (p->state->exit_fifo != NULL) {
+            efd = open_no_inherit("exit_fifo", O_WRONLY, -1);
+            if (efd < 0) {
+                write_message(g_log_fd, ERR_MSG, "open exit pipe failed:%d", SHIM_SYS_ERR(errno));
+                exit(EXIT_FAILURE);
+            }
+            p->exit_fd = efd;
+        }
+    }
+
+    // create main loop and start epoll
+    ret = process_io_init(p);
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "process io init failed:%d", ret);
+        exit(EXIT_FAILURE);
+    }
+
+    ret = open_io(p);
+    if (ret != SHIM_OK) {
+        exit(EXIT_FAILURE);
+    }
+
+    ret = create_process(p);
+    if (ret != SHIM_OK) {
+        exit(EXIT_FAILURE);
+    }
+
+    released_timeout_exit();
+
+    ret = process_signal_handle_routine(p);
+    if (ret != SHIM_OK) {
+        exit(EXIT_FAILURE);
+    }
+
+    exit(EXIT_SUCCESS);
+}

src/cmd/isulad-shim/process.h

+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
+ * iSulad licensed under the Mulan PSL v1.
+ * You can use this software according to the terms and conditions of the Mulan PSL v1.
+ * You may obtain a copy of Mulan PSL v1 at:
+ *     http://license.coscl.org.cn/MulanPSL
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v1 for more details.
+ * Author: leizhongkai
+ * Create: 2020-1-20
+ * Description: process definition
+ ******************************************************************************/
+
+#ifndef __SHIM_PROCESS_H_
+#define __SHIM_PROCESS_H_
+
+#include <pthread.h>
+#include <semaphore.h>
+#include <stdbool.h>
+#include "shim_client_process_state.h"
+
+enum {
+    stdid_in = 0,
+    stdid_out,
+    stdid_err
+};
+
+typedef struct {
+    int in;
+    int out;
+    int err;
+} stdio_t;
+
+typedef struct fd_node {
+    int fd;
+    struct fd_node *next;
+} fd_node_t;
+
+typedef struct {
+    int fd_from;
+    fd_node_t *fd_to;
+    int id;// 0,1,2
+    pthread_mutex_t mutex;
+} io_copy_t;
+
+typedef struct {
+    int epfd;
+    pthread_t tid;
+    pthread_attr_t attr;
+    sem_t sem_thd;
+    io_copy_t *ioc;
+    bool shutdown;
+} io_thread_t;
+
+typedef struct process {
+    char *id;
+    char *bundle;
+    char *runtime;
+    char *console_sock_path;
+    int io_loop_fd;
+    int exit_fd;
+    int ctr_pid;
+    stdio_t *stdio;
+    stdio_t *shim_io;
+    io_thread_t *io_threads[3];// stdin,stdout,stderr
+    shim_client_process_state *state;
+} process_t;
+
+typedef struct {
+    int listen_fd;
+    process_t *p;
+} console_accept_t;
+
+typedef struct {
+    int pid;
+    int status;
+} process_exit_t;
+
+
+
+process_t* new_process(char *id, char *bundle, char *runtime);
+
+int open_io(process_t *p);
+int process_io_init(process_t *p);
+int create_process(process_t *p);
+int process_signal_handle_routine(process_t *p);
+void process_delete(process_t *p);
+
+
+#endif
+