Show previous container logs (#2392)

* show previous container logs

* improved description of previous

docs/testing/logs/crashing.yaml

+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A manifest that creates a container that crashes after 100s and will be restarted by Kubernetes
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: crashing
+spec:
+  containers:
+    - name: crashing
+      image: alpine:3.4
+      command: ["/bin/sh", "-c", "for NUM in `seq 1 1 100`; do date; sleep 1; done"]
+  restartPolicy: Always

src/app/backend/handler/apihandler.go

@@ -2143,7 +2143,7 @@ func (apiHandler *APIHandler) handleLogs(request *restful.Request, response *res
 	if err != nil {
 		refLineNum = 0
 	}
-
+	usePreviousLogs := request.QueryParameter("previous") == "true"
 	offsetFrom, err1 := strconv.Atoi(request.QueryParameter("offsetFrom"))
 	offsetTo, err2 := strconv.Atoi(request.QueryParameter("offsetTo"))
 	logFilePosition := request.QueryParameter("logFilePosition")
@@ -2161,7 +2161,7 @@ func (apiHandler *APIHandler) handleLogs(request *restful.Request, response *res
 		}
 	}
 
-	result, err := container.GetLogDetails(k8sClient, namespace, podID, containerID, logSelector)
+	result, err := container.GetLogDetails(k8sClient, namespace, podID, containerID, logSelector, usePreviousLogs)
 	if err != nil {
 		handleInternalError(response, err)
 		return
@@ -2178,8 +2178,9 @@ func (apiHandler *APIHandler) handleLogFile(request *restful.Request, response *
 	namespace := request.PathParameter("namespace")
 	podID := request.PathParameter("pod")
 	containerID := request.PathParameter("container")
+	usePreviousLogs := request.QueryParameter("previous") == "true"
 
-	filename, logStream, err := container.GetLogFile(k8sClient, namespace, podID, containerID)
+	filename, logStream, err := container.GetLogFile(k8sClient, namespace, podID, containerID, usePreviousLogs)
 	if err != nil {
 		handleInternalError(response, err)
 		return

src/app/backend/resource/container/logs.go

@@ -53,10 +53,10 @@ func GetPodContainers(client kubernetes.Interface, namespace, podID string) (*Po
 	return containers, nil
 }
 
-// GetLogDetails returns logs for particular pod and container. When container
-// is null, logs for the first one are returned.
+// GetLogDetails returns logs for particular pod and container. When container is null, logs for the first one
+// are returned. Previous indicates to read archived logs created by log rotation or container crash
 func GetLogDetails(client kubernetes.Interface, namespace, podID string, container string,
-	logSelector *logs.Selection) (*logs.LogDetails, error) {
+	logSelector *logs.Selection, usePreviousLogs bool) (*logs.LogDetails, error) {
 	pod, err := client.CoreV1().Pods(namespace).Get(podID, metaV1.GetOptions{})
 	if err != nil {
 		return nil, err
@@ -66,7 +66,7 @@ func GetLogDetails(client kubernetes.Interface, namespace, podID string, contain
 		container = pod.Spec.Containers[0].Name
 	}
 
-	logOptions := mapToLogOptions(container, logSelector)
+	logOptions := mapToLogOptions(container, logSelector, usePreviousLogs)
 	rawLogs, err := readRawLogs(client, namespace, podID, logOptions)
 	if err != nil {
 		return nil, err
@@ -77,11 +77,11 @@ func GetLogDetails(client kubernetes.Interface, namespace, podID string, contain
 
 // Maps the log selection to the corresponding api object
 // Read limits are set to avoid out of memory issues
-func mapToLogOptions(container string, logSelector *logs.Selection) *v1.PodLogOptions {
+func mapToLogOptions(container string, logSelector *logs.Selection, previous bool) *v1.PodLogOptions {
 	logOptions := &v1.PodLogOptions{
 		Container:  container,
 		Follow:     false,
-		Previous:   false,
+		Previous:   previous,
 		Timestamps: true,
 	}
 
@@ -112,12 +112,13 @@ func readRawLogs(client kubernetes.Interface, namespace, podID string, logOption
 	return string(result), nil
 }
 
-// GetLogFile returns a stream to the log file which can be piped directly to the respose. This avoids oom issues.
-func GetLogFile(client kubernetes.Interface, namespace, podID string, container string) (string, io.ReadCloser, error) {
+// GetLogFile returns a stream to the log file which can be piped directly to the response. This avoids out of memory
+// issues. Previous indicates to read archived logs created by log rotation or container crash
+func GetLogFile(client kubernetes.Interface, namespace, podID string, container string, usePreviousLogs bool) (string, io.ReadCloser, error) {
 	logOptions := &v1.PodLogOptions{
 		Container:  container,
 		Follow:     false,
-		Previous:   false,
+		Previous:   usePreviousLogs,
 		Timestamps: false,
 	}
 	filename := fmt.Sprintf("logs-from-%v-in-%v.txt", container, podID)

src/app/backend/resource/container/logs_test.go

@@ -315,6 +315,32 @@ func TestGetLogs(t *testing.T) {
 				},
 			},
 		},
+		{
+			"don't try to split timestamp for error message",
+			"pod-1",
+			"an error message from api server",
+			"test",
+			logs.AllSelection,
+			&logs.LogDetails{
+				Info: logs.LogInfo{
+					PodName:       "pod-1",
+					ContainerName: "test",
+					FromDate:      "0",
+					ToDate:        "0",
+				},
+				LogLines: logs.LogLines{logs.LogLine{
+					Timestamp: "0",
+					Content:   "an error message from api server",
+				},},
+				Selection: logs.Selection{
+					ReferencePoint: logs.LogLineId{
+						LogTimestamp: "0",
+						LineNum:      1,
+					},
+					OffsetFrom: 0,
+					OffsetTo:   1},
+			},
+		},
 	}
 	for _, c := range cases {
 		actual := ConstructLogDetails(c.podId, c.rawLogs, c.container, c.logSelector)
@@ -356,7 +382,7 @@ func TestMapToLogOptions(t *testing.T) {
 		},
 	}
 	for _, c := range cases {
-		actual := mapToLogOptions(c.container, c.logSelector)
+		actual := mapToLogOptions(c.container, c.logSelector, false)
 		if !reflect.DeepEqual(actual, c.expected) {
 			t.Errorf("Test Case: %s.\nReceived: %#v \nExpected: %#v\n\n", c.info, actual, c.expected)
 		}

src/app/backend/resource/logs/logs.go

@@ -241,19 +241,20 @@ func (self LogLines) createLogLineId(lineIndex int) *LogLineId {
 	}
 }
 
-// ToLogLines converts rawLogs (string) to LogLines. This might be slow as we have to split ALL logs by \n.
-// The solution could be to split only required part of logs. To find reference line - do smart binary search on raw string -
-// select the middle, search slightly left and slightly right to find timestamp, eliminate half of the raw string,
-// repeat until found required timestamp. Later easily find and split N subsequent/preceding lines.
+// ToLogLines converts rawLogs (string) to LogLines. Proper log lines start with a timestamp which is chopped off.
+// In error cases the server returns a message without a timestamp
 func ToLogLines(rawLogs string) LogLines {
 	logLines := LogLines{}
 	for _, line := range strings.Split(rawLogs, "\n") {
 		if line != "" {
+			startsWithDate := ('0' <= line[0] && line[0] <= '9') //2017-...
 			idx := strings.Index(line, " ")
-			if idx > 0 {
+			if idx > 0 && startsWithDate {
 				timestamp := LogTimestamp(line[0:idx])
 				content := line[idx+1:]
 				logLines = append(logLines, LogLine{Timestamp: timestamp, Content: content})
+			} else {
+				logLines = append(logLines, LogLine{Timestamp: LogTimestamp("0"), Content: line})
 			}
 		}
 	}

src/app/frontend/logs/component.js

@@ -161,7 +161,6 @@ export class LogsController {
    */
   loadView(logFilePosition, referenceTimestamp, referenceLinenum, offsetFrom, offsetTo) {
     let namespace = this.stateParams_.objectNamespace;
-
     this.resource_(`api/v1/log/${namespace}/${this.pod}/${this.container}`)
         .get(
             {
@@ -170,6 +169,7 @@ export class LogsController {
               'referenceLineNum': referenceLinenum,
               'offsetFrom': offsetFrom,
               'offsetTo': offsetTo,
+              'previous': this.logsService.getPrevious(),
             },
             (podLogs) => {
               this.updateUiModel(podLogs);
@@ -294,7 +294,7 @@ export class LogsController {
   }
 
   /**
-   * Return the proper icon depending on the selection state
+   * Return the proper icon depending on the timestamp selection state
    * @export
    * @return {string}
    */
@@ -305,6 +305,18 @@ export class LogsController {
     return 'timer_off';
   }
 
+  /**
+   * Return the proper icon depending on the previous-container-logs selection state
+   * @export
+   * @return {string}
+   */
+  getPreviousIcon() {
+    if (this.logsService.getPrevious()) {
+      return 'exposure_neg_1';
+    }
+    return 'exposure_zero';
+  }
+
   /**
    * Return the link to download the log file
    * @export
@@ -312,7 +324,8 @@ export class LogsController {
    */
   getDownloadLink() {
     let namespace = this.stateParams_.objectNamespace;
-    return `/api/v1/log/file/${namespace}/${this.pod}/${this.container}`;
+    return `/api/v1/log/file/${namespace}/${this.pod}/${this.container}?previous=${
+        this.logsService.getPrevious()}`;
   }
 
   /**
@@ -357,6 +370,15 @@ export class LogsController {
     this.logsService.setShowTimestamp();
     this.logsSet = this.formatAllLogs_(this.podLogs.logs);
   }
+
+  /**
+   * Execute when a user changes the selected option for show previous container logs.
+   * @export
+   */
+  onPreviousChange() {
+    this.logsService.setPrevious();
+    this.loadNewest();
+  }
 }
 
 /**

src/app/frontend/logs/logs.html

@@ -60,6 +60,13 @@ limitations under the License.
           {{ctrl.getTimestampIcon()}}
         </md-icon>
       </md-button>
+      <md-button class="kd-logs-toolbar-button"
+                 ng-click="ctrl.onPreviousChange()">
+        <md-icon md-font-library="material-icons"
+                 class="kd-logs-icon">
+          {{ctrl.getPreviousIcon()}}
+        </md-icon>
+      </md-button>
       <md-button class="kd-logs-toolbar-button"
                  ng-href="{{ctrl.getDownloadLink()}}">
         <md-icon md-font-library="material-icons"

src/app/frontend/logs/service.js

@@ -29,6 +29,9 @@ export class LogsService {
 
     /** @private {boolean} */
     this.showTimestamp_ = false;
+
+    /** @private {boolean} */
+    this.previous_ = false;
   }
 
   /**
@@ -75,4 +78,19 @@ export class LogsService {
   getShowTimestamp() {
     return this.showTimestamp_;
   }
+
+  /**
+   * Switches the show previous flag
+   */
+  setPrevious() {
+    this.previous_ = !this.previous_;
+  }
+
+  /**
+   * Getter for the show previous flag
+   * @returns {boolean}
+   */
+  getPrevious() {
+    return this.previous_;
+  }
 }

src/test/frontend/logs/controller_test.js

@@ -132,7 +132,7 @@ describe('Logs controller', () => {
     expect(ctrl.logsSet.length).toEqual(3);
     httpBackend
         .expectGET(
-            'api/v1/log/namespace11/test-pod/container-name?logFilePosition=beginning&offsetFrom=25&offsetTo=125&referenceLineNum=11&referenceTimestamp=X')
+            'api/v1/log/namespace11/test-pod/container-name?logFilePosition=beginning&offsetFrom=25&offsetTo=125&previous=false&referenceLineNum=11&referenceTimestamp=X')
         .respond(200, otherLogs);
     httpBackend.flush();
     expect(ctrl.logsSet.length).toEqual(2);
@@ -145,7 +145,7 @@ describe('Logs controller', () => {
     expect(ctrl.logsSet.length).toEqual(3);
     httpBackend
         .expectGET(
-            'api/v1/log/namespace11/test-pod/container-name?logFilePosition=beginning&offsetFrom=-78&offsetTo=22&referenceLineNum=11&referenceTimestamp=X')
+            'api/v1/log/namespace11/test-pod/container-name?logFilePosition=beginning&offsetFrom=-78&offsetTo=22&previous=false&referenceLineNum=11&referenceTimestamp=X')
         .respond(200, otherLogs);
     httpBackend.flush();
     expect(ctrl.logsSet.length).toEqual(2);
@@ -158,7 +158,7 @@ describe('Logs controller', () => {
     expect(ctrl.logsSet.length).toEqual(3);
     httpBackend
         .expectGET(
-            'api/v1/log/namespace11/test-pod/container-name?logFilePosition=end&offsetFrom=2000000000&offsetTo=2000000100&referenceLineNum=0&referenceTimestamp=newest')
+            'api/v1/log/namespace11/test-pod/container-name?logFilePosition=end&offsetFrom=2000000000&offsetTo=2000000100&previous=false&referenceLineNum=0&referenceTimestamp=newest')
         .respond(200, otherLogs);
     httpBackend.flush();
     expect(ctrl.logsSet.length).toEqual(2);
@@ -172,7 +172,7 @@ describe('Logs controller', () => {
     expect(ctrl.logsSet.length).toEqual(3);
     httpBackend
         .expectGET(
-            'api/v1/log/namespace11/test-pod/container-name?logFilePosition=beginning&offsetFrom=-2000000100&offsetTo=-2000000000&referenceLineNum=0&referenceTimestamp=oldest')
+            'api/v1/log/namespace11/test-pod/container-name?logFilePosition=beginning&offsetFrom=-2000000100&offsetTo=-2000000000&previous=false&referenceLineNum=0&referenceTimestamp=oldest')
         .respond(200, otherLogs);
     httpBackend.flush();
     expect(ctrl.logsSet.length).toEqual(2);