support --log-opt params for oci runtime

Signed-off-by: Ngaohuatao <gaohuatao@huawei.com>

src/CMakeLists.txt

@@ -147,6 +147,7 @@ add_executable(isulad-shim
     ${ISULAD_SHIM_SRCS}
     ${CMAKE_BINARY_DIR}/json/shim_client_process_state.c
     ${CMAKE_BINARY_DIR}/json/json_common.c
+    ${CMAKE_BINARY_DIR}/json/logger_json_file.c
     ${commonjsonsrcs}
     )
 target_include_directories(isulad-shim PUBLIC ${ISULAD_SHIM_INCS} ${SHARED_INCS})

src/cmd/isulad-shim/process.c

@@ -35,9 +35,11 @@
 
 #include "common.h"
 #include "process.h"
+#include "terminal.h"
 
 #define MAX_EVENTS 100
 #define DEFAULT_IO_COPY_BUF (16*1024)
+#define DEFAULT_LOG_FILE_SIZE (4*1024)
 
 extern int g_log_fd;
 
@@ -143,6 +145,10 @@ static int add_io_dispatch(int epfd, io_thread_t *io_thd, int from, int to)
         return SHIM_ERR;
     }
     fn->fd = to;
+    fn->is_log = false;
+    if (io_thd->terminal != NULL && to == io_thd->terminal->fd) {
+        fn->is_log = true;
+    }
     fn->next = NULL;
 
     pthread_mutex_lock(&(ioc->mutex));
@@ -254,11 +260,15 @@ static void* task_io_copy(void *data)
         } else {
             fd_node_t *fn = ioc->fd_to;
             for (; fn != NULL; fn = fn->next) {
-                int w_count;
-                w_count = write_nointr(fn->fd, buf, r_count);
-                if (w_count < 0) {
-                    // remove the write fd
-                    remove_io_dispatch(io_thd, -1, fn->fd);
+                if (fn->is_log) {
+                    shim_write_container_log_file(io_thd->terminal, ioc->id == stdid_out ? "stdout" : "stderr", buf, r_count);
+                } else {
+                    int w_count = 0;
+                    w_count = write_nointr(fn->fd, buf, r_count);
+                    if (w_count < 0) {
+                        // remove the write fd
+                        remove_io_dispatch(io_thd, -1, fn->fd);
+                    }
                 }
             }
         }
@@ -318,6 +328,8 @@ static int process_io_start(process_t *p, int std_id)
     io_thd->epfd = p->io_loop_fd;
     io_thd->ioc = ioc;
     io_thd->shutdown = false;
+    io_thd->terminal = std_id != stdid_in ? p->terminal : NULL;
+
     p->io_threads[std_id] = io_thd;
 
     ret = pthread_create(&(io_thd->tid), NULL, task_io_copy, io_thd);
@@ -398,6 +410,9 @@ static int connect_to_isulad(process_t *p, int std_id, const char *isulad_stdio,
     }
 
     if (*fd_from != -1) {
+        if (std_id != stdid_in && p->io_threads[std_id]->terminal != NULL) {
+            (void)add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, p->terminal->fd);
+        }
         return add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, *fd_to);
     }
 
@@ -677,11 +692,61 @@ static void adapt_for_isulad_stdin(process_t *p)
     }
 }
 
+static int terminal_init(log_terminal **terminal, shim_client_process_state *p_state)
+{
+    log_terminal *log_term = NULL;
+
+    log_term = calloc(1, sizeof(log_terminal));
+    if (log_term == NULL) {
+        write_message(g_log_fd, ERR_MSG, "Failed to calloc log_terminal");
+        goto clean_out;
+    }
+
+    if (pthread_rwlock_init(&log_term->log_terminal_rwlock, NULL) != 0) {
+        write_message(g_log_fd, ERR_MSG, "Failed to init isulad conf rwlock");
+        goto clean_out;
+    }
+
+    if (p_state == NULL) {
+        goto clean_out;
+    }
+
+    log_term->log_path = p_state->log_path;
+    /* Default to disable log. */
+    log_term->fd = -1;
+    log_term->log_maxfile = 1;
+    /* Default value 4k, the min size of a single log file */
+    log_term->log_maxsize = DEFAULT_LOG_FILE_SIZE;
+
+    if (p_state->log_maxfile > log_term->log_maxfile) {
+        log_term->log_maxfile = (unsigned int)p_state->log_maxfile;
+    }
+
+    if (p_state->log_maxsize > log_term->log_maxsize) {
+        log_term->log_maxsize = (uint64_t)p_state->log_maxsize;
+    }
+
+    if (log_term->log_path != NULL) {
+        if (shim_create_container_log_file(log_term)) {
+            goto clean_out;
+        }
+    }
+
+    *terminal = log_term;
+
+    return SHIM_OK;
+clean_out:
+    free(log_term);
+    *terminal = NULL;
+    return SHIM_ERR;
+}
+
 process_t* new_process(char *id, char *bundle, char *runtime)
 {
     shim_client_process_state* p_state;
     process_t* p = NULL;
     int i;
+    int ret;
 
     p_state = load_process();
     if (p_state == NULL) {
@@ -692,6 +757,13 @@ process_t* new_process(char *id, char *bundle, char *runtime)
     if (p == NULL) {
         return NULL;
     }
+    ret = terminal_init(&(p->terminal), p_state);
+    if (ret != SHIM_OK) {
+        free(p);
+        p = NULL;
+        return p;
+    }
+
     p->id = id;
     p->bundle = bundle;
     p->runtime = runtime;
@@ -703,6 +775,7 @@ process_t* new_process(char *id, char *bundle, char *runtime)
     p->ctr_pid = -1;
     p->stdio = NULL;
     p->shim_io = NULL;
+
     for (i = 0; i < 3; i ++) {
         p->io_threads[i] = NULL;
     }
@@ -726,7 +799,6 @@ int open_io(process_t *p)
     return open_generic_io(p);
 }
 
-
 int process_io_init(process_t *p)
 {
     int ret = SHIM_ERR;

src/cmd/isulad-shim/process.h

@@ -20,6 +20,7 @@
 #include <semaphore.h>
 #include <stdbool.h>
 #include "shim_client_process_state.h"
+#include "terminal.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -39,6 +40,7 @@ typedef struct {
 
 typedef struct fd_node {
     int fd;
+    bool is_log;
     struct fd_node *next;
 } fd_node_t;
 
@@ -56,6 +58,7 @@ typedef struct {
     sem_t sem_thd;
     io_copy_t *ioc;
     bool shutdown;
+    log_terminal *terminal;
 } io_thread_t;
 
 typedef struct process {
@@ -66,6 +69,7 @@ typedef struct process {
     int io_loop_fd;
     int exit_fd;
     int ctr_pid;
+    log_terminal *terminal;
     stdio_t *stdio;
     stdio_t *shim_io;
     io_thread_t *io_threads[3];// stdin,stdout,stderr

src/cmd/isulad-shim/terminal.c

+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+ * iSulad licensed under the Mulan PSL v1.
+ * You can use this software according to the terms and conditions of the Mulan PSL v1.
+ * You may obtain a copy of Mulan PSL v1 at:
+ *     http://license.coscl.org.cn/MulanPSL
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v1 for more details.
+ * Author: gaohuatao
+ * Create: 2020-3-9
+ * Description: container logs ops
+ ******************************************************************************/
+#define _GNU_SOURCE
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <limits.h>
+#include "terminal.h"
+#include "common.h"
+
+#define BUF_CACHE_SIZE (16 * 1024)
+
+static ssize_t shim_write_nointr_lock(log_terminal *terminal, const void *buf, size_t count)
+{
+    ssize_t ret;
+
+    (void)pthread_rwlock_wrlock(&terminal->log_terminal_rwlock);
+    ret = write_nointr(terminal->fd, buf, count);
+    (void)pthread_rwlock_unlock(&terminal->log_terminal_rwlock);
+
+    return ret;
+}
+
+static int shim_rename_old_log_file(log_terminal *terminal)
+{
+    int ret;
+    unsigned int i;
+    char tmp[PATH_MAX] = {0};
+    char *rename_fname = NULL;
+
+    for (i = terminal->log_maxfile - 1; i > 1; i--) {
+        ret = snprintf(tmp, PATH_MAX, "%s.%u", terminal->log_path, i);
+        if (ret < 0 || ret >= PATH_MAX) {
+            free(rename_fname);
+            return SHIM_ERR;
+        }
+        free(rename_fname);
+        rename_fname = safe_strdup(tmp);
+
+        ret = snprintf(tmp, PATH_MAX, "%s.%u", terminal->log_path, (i - 1));
+        if (ret < 0 || ret >= PATH_MAX) {
+            free(rename_fname);
+            return SHIM_ERR;
+        }
+
+        ret = rename(tmp, rename_fname);
+        if (ret < 0 && errno != ENOENT) {
+            free(rename_fname);
+            return SHIM_ERR;
+        }
+    }
+
+    free(rename_fname);
+    return SHIM_OK;
+}
+
+static int shim_dump_log_file(log_terminal *terminal)
+{
+    int ret;
+    char *file_newname;
+    size_t len_path = strlen(terminal->log_path) + sizeof(".1");
+
+    /* isulad: rotate old log file first */
+    ret = shim_rename_old_log_file(terminal);
+    if (ret != 0) {
+        return SHIM_ERR;
+    }
+
+    file_newname = calloc(len_path, 1);
+    if (file_newname == NULL) {
+        return SHIM_ERR;
+    }
+
+    ret = snprintf(file_newname, len_path, "%s.1", terminal->log_path);
+    if (ret < 0 || (size_t)ret >= len_path) {
+        ret = -1;
+        goto clean_out;
+    }
+
+    /*
+     * Rename the file console.log to console.log.1 then create and open console.log again.
+     * fd points to console.log file always.
+     */
+    close(terminal->fd);
+    terminal->fd = -1;
+    (void)rename(terminal->log_path, file_newname);
+    ret = shim_create_container_log_file(terminal);
+clean_out:
+    free(file_newname);
+    return ret;
+}
+
+static int64_t get_log_file_size(int fd)
+{
+    struct stat log_st;
+    int ret;
+
+    ret = fstat(fd, &log_st);
+    if (ret < 0) {
+        return SHIM_ERR;
+    }
+
+    if (S_IFREG != (log_st.st_mode & S_IFMT)) {
+        return SHIM_ERR;
+    }
+
+    return log_st.st_size;
+}
+
+static int shim_json_data_write(log_terminal *terminal, const char *buf,
+                                int read_count)
+{
+    int ret;
+    int64_t available_space = -1;
+    int64_t file_size;
+
+    file_size = get_log_file_size(terminal->fd);
+    if (file_size < 0) {
+        return SHIM_ERR;
+    }
+
+    available_space = terminal->log_maxsize - file_size;
+    if (read_count <= available_space) {
+        return shim_write_nointr_lock(terminal, buf, read_count);
+    }
+
+    ret = shim_dump_log_file(terminal);
+    if (ret < 0) {
+        return SHIM_ERR;
+    }
+
+    /*
+     * Now file is new, then write the max bytes that will be wrote to log file.
+     * We have set the log file min size 16k, so the scenario of log_maxsize < read_count
+     * shouldn't happen, otherwise, discard some last bytes.
+     */
+    ret = shim_write_nointr_lock(terminal, buf,
+                                 terminal->log_maxsize < read_count ? terminal->log_maxsize : read_count);
+    if (ret < 0) {
+        return SHIM_ERR;
+    }
+
+    return (read_count - ret);
+}
+
+static bool get_time_buffer(struct timespec *timestamp, char *timebuffer,
+                            size_t maxsize)
+{
+    struct tm tm_utc = { 0 };
+    int32_t nanos = 0;
+    time_t seconds;
+    size_t len = 0;
+    int ret = 0;
+
+    if (!timebuffer || !maxsize) {
+        return false;
+    }
+
+    seconds = (time_t)timestamp->tv_sec;
+    gmtime_r(&seconds, &tm_utc);
+    strftime(timebuffer, maxsize, "%Y-%m-%dT%H:%M:%S", &tm_utc);
+
+    nanos = (int32_t)timestamp->tv_nsec;
+    len = strlen(timebuffer);
+    ret = snprintf(timebuffer + len, (maxsize - len), ".%09dZ", nanos);
+    if (ret < 0 || ret >= (maxsize - len)) {
+        return false;
+    }
+
+    return true;
+}
+
+static bool get_now_time_buffer(char *timebuffer, size_t maxsize)
+{
+    int err = 0;
+    struct timespec ts;
+
+    err = clock_gettime(CLOCK_REALTIME, &ts);
+    if (err != 0) {
+        return false;
+    }
+
+    return get_time_buffer(&ts, timebuffer, maxsize);
+}
+
+static ssize_t shim_logger_write(log_terminal *terminal, const char *type, const char *buf,
+                                 int read_count)
+{
+    logger_json_file *msg = NULL;
+    ssize_t ret = -1;
+    size_t len;
+    char *json = NULL;
+    char timebuffer[64] = { 0 };
+    parser_error err = NULL;
+    struct parser_context ctx = { OPT_GEN_SIMPLIFY | OPT_GEN_NO_VALIDATE_UTF8, stderr };
+
+    if (read_count < 0 || read_count >= INT_MAX) {
+        return SHIM_ERR;
+    }
+
+    msg = calloc(sizeof(logger_json_file), 1);
+    if (msg == NULL) {
+        return SHIM_ERR;
+    }
+
+    msg->log = calloc(read_count, 1);
+    if (!msg->log) {
+        goto cleanup;
+    }
+    memcpy(msg->log, buf, read_count);
+    msg->log_len = read_count;
+    msg->stream = type ? safe_strdup(type) : safe_strdup("stdout");
+
+    get_now_time_buffer(timebuffer, sizeof(timebuffer));
+    msg->time = safe_strdup(timebuffer);
+    json = logger_json_file_generate_json(msg, &ctx, &err);
+    if (!json) {
+        goto cleanup;
+    }
+    len = strlen(json);
+    json[len] = '\n';
+    if (terminal->fd < 0) {
+        goto cleanup;
+    }
+
+    ret = shim_json_data_write(terminal, json, len + 1);
+cleanup:
+    free(json);
+    free_logger_json_file(msg);
+    free(err);
+    return ret;
+}
+
+void shim_write_container_log_file(log_terminal *terminal, const char *type, char *buf,
+                                   int read_count)
+{
+    static char cache[BUF_CACHE_SIZE];
+    static int size = 0;
+    int upto, index;
+    int begin = 0, buf_readed = 0,  buf_left = 0;
+
+    if (terminal == NULL) {
+        return;
+    }
+
+    if (buf != NULL && read_count > 0) {
+        upto = size + read_count;
+        if (upto > BUF_CACHE_SIZE) {
+            upto = BUF_CACHE_SIZE;
+        }
+
+        if (upto > size) {
+            buf_readed = upto - size;
+            memcpy(cache + size, buf, buf_readed);
+            buf_left = read_count - buf_readed;
+            size += buf_readed;
+        }
+    }
+
+    if (size == 0) {
+        return;
+    }
+
+    for (index = 0; index < size; index++) {
+        if (cache[index] == '\n') {
+            (void)shim_logger_write(terminal, type, cache + begin, index - begin + 1);
+            begin = index + 1;
+        }
+    }
+
+    if (buf == NULL || (begin == 0 && size == BUF_CACHE_SIZE)) {
+        if (begin < size) {
+            (void)shim_logger_write(terminal, type, cache + begin, size - begin);
+            begin = 0;
+            size = 0;
+        }
+        if (buf == NULL) {
+            return;
+        }
+    }
+
+    if (begin > 0) {
+        memcpy(cache, cache + begin, size - begin);
+        size -= begin;
+    }
+
+    if (buf_left > 0) {
+        memcpy(cache + size, buf + buf_readed, buf_left);
+        size += buf_left;
+    }
+}
+
+int shim_create_container_log_file(log_terminal *terminal)
+{
+    if (!terminal->log_path) {
+        return SHIM_ERR;
+    }
+
+    terminal->fd = open(terminal->log_path, O_CLOEXEC | O_RDWR | O_CREAT | O_APPEND, 0600);
+    if (terminal->fd < 0) {
+        return SHIM_ERR;
+    }
+
+    return SHIM_OK;
+}
+

src/cmd/isulad-shim/terminal.h

+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+ * iSulad licensed under the Mulan PSL v1.
+ * You can use this software according to the terms and conditions of the Mulan PSL v1.
+ * You may obtain a copy of Mulan PSL v1 at:
+ *     http://license.coscl.org.cn/MulanPSL
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v1 for more details.
+ * Author: gaohuatao
+ * Create: 2020-3-9
+ * Description: container logs ops
+ ******************************************************************************/
+
+#ifndef __SHIM_TERMINAL_H
+#define __SHIM_TERMINAL_H
+
+#include <pthread.h>
+#include <unistd.h>
+#include "logger_json_file.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef PATH_MAX
+#define PATH_MAX 4096
+#endif
+
+typedef struct {
+    uint64_t log_maxsize;
+    char *log_path;
+    int fd;
+    unsigned int log_maxfile;
+    pthread_rwlock_t log_terminal_rwlock;
+} log_terminal;
+
+void shim_write_container_log_file(log_terminal *terminal, const char *type, char *buf,
+                                   int bytes_read);
+
+int shim_create_container_log_file(log_terminal *terminal);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __SHIM_TERMINAL_H */

src/json/schema/schema/shim/client/process-state.json

@@ -188,6 +188,15 @@
         },
         "rootGID": {
             "type": "integer"
+        },
+        "logPath": {
+            "type": "string"
+        },
+        "logMaxfile": {
+            "type": "integer"
+        },
+        "logMaxsize": {
+            "type": "int64"
         }
     }
 }

src/runtime/isula/isula_rt_ops.c

@@ -28,11 +28,14 @@
 #include "shim_client_process_state.h"
 #include "oci_runtime_state.h"
 #include "isulad_config.h"
+#include "utils_string.h"
+#include "libisulad.h"
 
 #define SHIM_BINARY "isulad-shim"
 #define SHIM_LOG_SIZE ((BUFSIZ-100)/2)
 #define PID_WAIT_TIME 120
 
+
 static void copy_process(shim_client_process_state *p, defs_process *dp)
 {
     p->args = dp->args;
@@ -52,6 +55,31 @@ static void copy_process(shim_client_process_state *p, defs_process *dp)
     p->rlimits_len = dp->rlimits_len;
 }
 
+static void copy_annotations(shim_client_process_state *p, json_map_string_string *anno)
+{
+    size_t i;
+    if (anno == NULL) {
+        return;
+    }
+    for (i = 0; i < anno->len; i++) {
+        if (strcmp(anno->keys[i], CONTAINER_LOG_CONFIG_KEY_FILE) == 0) {
+            p->log_path = anno->values[i];
+        } else if (strcmp(anno->keys[i], CONTAINER_LOG_CONFIG_KEY_ROTATE) == 0) {
+            int tmaxfile = 0;
+            if (util_safe_int(anno->values[i], &tmaxfile) == 0 && tmaxfile > 0) {
+                p->log_maxfile = tmaxfile;
+            }
+            continue;
+        }
+        if (strcmp(anno->keys[i], CONTAINER_LOG_CONFIG_KEY_SIZE) == 0) {
+            int64_t tmaxsize = 0;
+            if (util_parse_byte_size_string(anno->values[i], &tmaxsize) == 0 && tmaxsize > 0) {
+                p->log_maxsize = tmaxsize;
+            }
+        }
+    }
+}
+
 static int file_write_int(const char *fname, int val)
 {
     char sint[UINT_LEN] = {0};
@@ -731,6 +759,7 @@ int rt_isula_create(const char *id, const char *runtime,
     p.runtime_args = (char **)runtime_args;
     p.runtime_args_len = runtime_args_len;
     copy_process(&p, config->process);
+    copy_annotations(&p, config->annotations);
 
     ret = create_process_json_file(workdir, &p);
     if (ret != 0) {