[FLINK-2357] [web dashboard] Auto-update overview page

flink-runtime-web/README.md

@@ -113,7 +113,7 @@ mvn -DskipTests clean package
 To simplify continuous development, one can use a *standalone proxy server*, together with automatic
 re-compilation:
 
-1. Edit the file `app/scripts/index.coffee`. Comment/uncomment the lines that define the `webServer`, `jobServer`, and `newServer` URLs.
+1. Edit the file `app/scripts/index.coffee`. Comment/uncomment the lines that define the `webServer`, and `jobServer` URLs.
 2. Re-compile the files via `gulp`. By calling `gulp watch`, the build-tool autocompiles future changes.
 3. Start the proxy server via `node server.js`
 4. Access the dashboard at [`http://localhost:3000`](http://localhost:3000)

flink-runtime-web/web-dashboard/app/partials/jobs/completed-jobs.jade

@@ -38,7 +38,7 @@ nav.navbar.navbar-default.navbar-fixed-top.navbar-main
       tr(ng-repeat="job in jobs" ui-sref="single-job.plan({ jobid: job.jid })")
         td {{job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}
         td {{job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}
-        td {{job['end-time'] - job['start-time']}} ms
+        td {{job.duration}} ms
         td {{job.name}}
         td {{job.jid}}
         td.label-group

flink-runtime-web/web-dashboard/app/partials/jobs/job.jade

@@ -32,11 +32,12 @@ nav.navbar.navbar-default.navbar-fixed-top.navbar-main(ng-if="job")
 
   .navbar-info.first.last.hidden-xs.hidden-sm
     | {{ job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss' }}
-    | - 
-    | {{ job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss' }}
+    span(ng-if="job['end-time'] > -1")
+      | - 
+      | {{ job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss' }}
   
-  .navbar-info.last.first
-    | {{job['end-time'] - job['start-time']}} ms
+  .navbar-info.last.first(ng-if="job.duration > -1")
+    | {{job.duration}} ms
 
 nav.navbar.navbar-default.navbar-fixed-top.navbar-main-additional(ng-if="job")
   ul.nav.nav-tabs

flink-runtime-web/web-dashboard/app/partials/jobs/job.plan.node.jade

@@ -25,7 +25,7 @@
 
     .panel-info(ng-if="node.description")
       .label-group
-        bs-label(status="{{status}}" ng-repeat="(index, status) in stateList") {{node.vertex.groupvertex[status]}}
+        bs-label(status="{{status}}" ng-repeat="(index, status) in stateList") {{node.vertex.tasks[status]}}
 
   .panel-heading.clearfix
     .panel-info.first.last(ng-if="node.description")
@@ -40,7 +40,7 @@
 
       tbody
         tr(ng-repeat="vertex in node.vertex.groupvertex.groupmembers")
-          td {{ vertex.vertexname | humanizeTaskName }}
+          td {{ vertex.name | humanizeTaskName }}
           td 
             bs-label(status="{{vertex.vertexstatus}}") {{vertex.vertexstatus}}
 

flink-runtime-web/web-dashboard/app/partials/jobs/running-jobs.jade

@@ -27,8 +27,8 @@ nav.navbar.navbar-default.navbar-fixed-top.navbar-main
     thead
       tr
         th Start Time
-        th End Time
-        th Duration
+        //- th End Time
+        //- th Duration
         th Job Name
         th Job ID
         th Tasks
@@ -37,8 +37,8 @@ nav.navbar.navbar-default.navbar-fixed-top.navbar-main
     tbody
       tr(ng-repeat="job in jobs" ui-sref="single-job.plan({ jobid: job.jid })")
         td {{job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}
-        td {{job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}
-        td {{job['end-time'] - job['start-time']}} ms
+        //- td {{job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}
+        //- td {{job.duration}} ms
         td {{job.name}}
         td {{job.jid}}
         td.label-group

flink-runtime-web/web-dashboard/app/partials/overview.jade

@@ -87,8 +87,8 @@ nav.navbar.navbar-default.navbar-fixed-top.navbar-main
         thead
           tr
             th Start Time
-            th End Time
-            th Duration
+            //- th End Time
+            //- th Duration
             th Job Name
             th Job ID
             th Tasks
@@ -97,8 +97,8 @@ nav.navbar.navbar-default.navbar-fixed-top.navbar-main
         tbody
           tr(ng-repeat="job in runningJobs" ui-sref="single-job.plan({ jobid: job.jid })")
             td {{job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}
-            td {{job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}
-            td {{job.duration}}
+            //- td {{job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}
+            //- td {{job.duration}} ms
             td {{job.name}}
             td {{job.jid}}
             td.label-group
@@ -126,7 +126,7 @@ nav.navbar.navbar-default.navbar-fixed-top.navbar-main
           tr(ng-repeat="job in finishedJobs" ui-sref="single-job.plan({ jobid: job.jid })")
             td {{job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}
             td {{job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}
-            td {{job['end-time'] - job['start-time']}} ms
+            td {{job.duration}} ms
             td {{job.name}}
             td {{job.jid}}
             td.label-group

flink-runtime-web/web-dashboard/app/scripts/common/services.coffee

@@ -24,7 +24,7 @@ angular.module('flinkApp')
   @loadConfig = ->
     deferred = $q.defer()
 
-    $http.get flinkConfig.newServer + "/config"
+    $http.get flinkConfig.jobServer + "/config"
     .success (data, status, headers, config) ->
       deferred.resolve(data)
 

flink-runtime-web/web-dashboard/app/scripts/index.coffee

@@ -30,9 +30,7 @@ angular.module('flinkApp', ['ui.router', 'angularMoment'])
 
 .value 'flinkConfig', {
   jobServer: 'http://localhost:8081'
-  newServer: 'http://localhost:8081'
 #  jobServer: 'http://localhost:3000/new-server'
-#  newServer: 'http://localhost:3000/new-server'
   "refresh-interval": 10000
 }
 

flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.ctrl.coffee

@@ -70,5 +70,5 @@ angular.module('flinkApp')
 # --------------------------------------
 
 .controller 'JobTimelineVertexController', ($scope, $state, $stateParams, JobsService) ->
-  JobsService.getVertex($stateParams.jobid, $stateParams.vertexId).then (data) ->
+  JobsService.getVertex($stateParams.vertexId).then (data) ->
     $scope.vertex = data

flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.dir.coffee

@@ -35,34 +35,32 @@ angular.module('flinkApp')
     analyzeTime = (data) ->
       testData = []
 
-      angular.forEach data.groupvertex.groupmembers, (vertex, i) ->
-        vTime = data.verticetimes[vertex.vertexid]
-
+      angular.forEach data.subtasks, (subtask, i) ->
         testData.push {
-          label: "#{vertex.vertexinstancename} (#{i})"
+          label: "#{subtask.host} (#{subtask.subtask})"
           times: [
             {
               label: "Scheduled"
               color: "#666"
               borderColor: "#555"
-              starting_time: vTime["SCHEDULED"] * 100
-              ending_time: vTime["DEPLOYING"] * 100
+              starting_time: subtask.timestamps["SCHEDULED"]
+              ending_time: subtask.timestamps["DEPLOYING"]
               type: 'regular'
             }
             {
               label: "Deploying"
               color: "#aaa"
               borderColor: "#555"
-              starting_time: vTime["DEPLOYING"] * 100
-              ending_time: vTime["RUNNING"] * 100
+              starting_time: subtask.timestamps["DEPLOYING"]
+              ending_time: subtask.timestamps["RUNNING"]
               type: 'regular'
             }
             {
               label: "Running"
               color: "#ddd"
               borderColor: "#555"
-              starting_time: vTime["RUNNING"] * 100
-              ending_time: vTime["FINISHED"] * 100
+              starting_time: subtask.timestamps["RUNNING"]
+              ending_time: subtask.timestamps["FINISHED"]
               type: 'regular'
             }
           ]
@@ -115,26 +113,21 @@ angular.module('flinkApp')
           label: "Scheduled"
           color: "#cccccc"
           borderColor: "#555"
-          starting_time: data.oldV["SCHEDULED"]
-          ending_time: data.oldV["SCHEDULED"] + 1
-          # link: vertex.groupvertexid
+          starting_time: data.timestamps["CREATED"]
+          ending_time: data.timestamps["CREATED"] + 1
           type: 'scheduled'
         ]
 
 
-      angular.forEach data.oldV.groupvertices, (vertex) ->
-        vTime = data.oldV.groupverticetimes[vertex.groupvertexid]
-
-        # console.log vTime, vertex.groupvertexid
-
+      angular.forEach data.vertices, (vertex) ->
         testData.push 
           times: [
-            label: translateLabel(vertex.groupvertexname)
+            label: translateLabel(vertex.name)
             color: "#d9f1f7"
             borderColor: "#62cdea"
-            starting_time: vTime["STARTED"]
-            ending_time: vTime["ENDED"]
-            link: vertex.groupvertexid
+            starting_time: vertex['start-time']
+            ending_time: vertex['end-time']
+            link: vertex.id
             type: 'regular'
           ]
 

flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.svc.coffee

@@ -69,7 +69,7 @@ angular.module('flinkApp')
   @listJobs = ->
     deferred = $q.defer()
 
-    $http.get flinkConfig.newServer + "/joboverview"
+    $http.get flinkConfig.jobServer + "/joboverview"
     .success (data, status, headers, config) ->
 
       angular.forEach data, (list, listKey) ->
@@ -80,11 +80,6 @@ angular.module('flinkApp')
           when 'cancelled' then jobs.cancelled = list
           when 'failed' then jobs.failed = list
 
-        # angular.forEach list, (jobid, index) ->
-        #   $http.get flinkConfig.newServer + "/jobs/" + jobid
-        #   .success (details) ->
-        #     list[index] = details
-
       deferred.resolve(jobs)
       notifyObservers()
 
@@ -100,23 +95,28 @@ angular.module('flinkApp')
     currentJob = null
     deferreds.job = $q.defer()
 
-    $http.get flinkConfig.newServer + "/jobs/" + jobid
+    $http.get flinkConfig.jobServer + "/jobs/" + jobid
     .success (data, status, headers, config) ->
-      data.time = Date.now()
+      # data.time = Date.now()
 
-      $http.get flinkConfig.newServer + "/jobs/" + jobid + "/vertices"
+      $http.get flinkConfig.jobServer + "/jobs/" + jobid + "/vertices"
       .success (vertices) ->
         data = angular.extend(data, vertices)
 
-        $http.get flinkConfig.jobServer + "/jobsInfo?get=job&job=" + jobid
-        .success (oldVertices) ->
-          data.oldV = oldVertices[0]
+        # $http.get flinkConfig.jobServer + "/jobsInfo?get=job&job=" + jobid
+        # .success (oldVertices) ->
+        #   data.oldV = oldVertices[0]
+
+        $http.get flinkConfig.jobServer + "/jobs/" + jobid + "/config"
+        .success (jobConfig) ->
+          data = angular.extend(data, jobConfig)
 
-          $http.get flinkConfig.jobServer + "/jobs/" + jobid + "/config"
-          .success (jobConfig) ->
-            data = angular.extend(data, jobConfig)
+          $http.get flinkConfig.jobServer + "/jobs/" + jobid + "/exceptions"
+          .success (exceptions) ->
+            data.exceptions = exceptions
 
             currentJob = data
+
             deferreds.job.resolve(data)
 
     deferreds.job.promise
@@ -125,7 +125,7 @@ angular.module('flinkApp')
     currentPlan = null
     deferreds.plan = $q.defer()
 
-    $http.get flinkConfig.newServer + "/jobs/" + jobid + "/plan"
+    $http.get flinkConfig.jobServer + "/jobs/" + jobid + "/plan"
     .success (data) ->
       currentPlan = data
 
@@ -142,33 +142,38 @@ angular.module('flinkApp')
 
       null
 
-    deferred = $q.defer()
 
-    # if currentPlan
-    #   deferred.resolve(seekNode(nodeid, currentPlan.nodes))
-    # else
-    #   # deferreds.plan.promise.then (data) ->
-    #   $q.all([deferreds.plan.promise, deferreds.job.promise]).then (data) ->
-    #     console.log 'resolving getNode'
-    #     deferred.resolve(seekNode(nodeid, currentPlan.nodes))
+    deferred = $q.defer()
 
     $q.all([deferreds.plan.promise, deferreds.job.promise]).then (data) =>
       foundNode = seekNode(nodeid, currentPlan.nodes)
 
-      # @getVertex(currentJob.jid, currentJob.oldV.groupvertices[0].groupvertexid).then (vertex) ->
-      @getVertex(currentJob.jid, nodeid).then (vertex) ->
-        foundNode.vertex = vertex
-        deferred.resolve(foundNode)
+      # $http.get flinkConfig.jobServer + "/jobsInfo?get=groupvertex&job=" + currentJob.jid + "&groupvertex=" + nodeid
+
+      foundNode.vertex = @seekVertex(nodeid)
+
+      deferred.resolve(foundNode)
 
     deferred.promise
 
+  @seekVertex = (nodeid) ->
+    for vertex in currentJob.vertices
+      return vertex if vertex.id is nodeid
+
+    return null
+
 
-  @getVertex = (jobId, vertexId) ->
+  @getVertex = (vertexid) ->
     deferred = $q.defer()
 
-    $http.get flinkConfig.jobServer + "/jobsInfo?get=groupvertex&job=" + jobId + "&groupvertex=" + vertexId
-    .success (data) ->
-      deferred.resolve(data)
+    $q.all([deferreds.job.promise]).then (data) =>
+      vertex = @seekVertex(vertexid)
+
+      $http.get flinkConfig.jobServer + "/jobs/" + currentJob.jid + "/vertices/" + vertexid + "/subtasktimes"
+      .success (data) ->
+        vertex.subtasks = data.subtasks
+
+        deferred.resolve(vertex)
 
     deferred.promise
 

flink-runtime-web/web-dashboard/app/scripts/modules/overview/overview.ctrl.coffee

@@ -18,7 +18,7 @@
 
 angular.module('flinkApp')
 
-.controller 'OverviewController', ($scope, OverviewService, JobsService) ->
+.controller 'OverviewController', ($scope, OverviewService, JobsService, $interval, flinkConfig) ->
   $scope.jobObserver = ->
     $scope.runningJobs = JobsService.getJobs('running')
     $scope.finishedJobs = JobsService.getJobs('finished')
@@ -31,3 +31,11 @@ angular.module('flinkApp')
 
   OverviewService.loadOverview().then (data) ->
     $scope.overview = data
+
+  refresh = $interval ->
+    OverviewService.loadOverview().then (data) ->
+      $scope.overview = data
+  , flinkConfig["refresh-interval"]
+
+  $scope.$on '$destroy', ->
+    $interval.cancel(refresh)

flink-runtime-web/web-dashboard/web/partials/jobs/completed-jobs.html

@@ -38,7 +38,7 @@ limitations under the License.
       <tr ng-repeat="job in jobs" ui-sref="single-job.plan({ jobid: job.jid })">
         <td>{{job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}</td>
         <td>{{job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}</td>
-        <td>{{job['end-time'] - job['start-time']}} ms</td>
+        <td>{{job.duration}} ms</td>
         <td>{{job.name}}</td>
         <td>{{job.jid}}</td>
         <td class="label-group">

flink-runtime-web/web-dashboard/web/partials/jobs/job.html

@@ -28,12 +28,10 @@ limitations under the License.
       <bs-label status="{{status}}" ng-repeat="(status, value) in job.tasks">{{value}}</bs-label>
     </div>
   </div>
-  <div class="navbar-info first last hidden-xs hidden-sm">
-    {{ job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss' }}
-    - 
-    {{ job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss' }}
-  </div>
-  <div class="navbar-info last first">{{job['end-time'] - job['start-time']}} ms</div>
+  <div class="navbar-info first last hidden-xs hidden-sm">{{ job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss' }}<span ng-if="job['end-time'] &gt; -1">
+      - 
+      {{ job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss' }}</span></div>
+  <div ng-if="job.duration &gt; -1" class="navbar-info last first">{{job.duration}} ms</div>
 </nav>
 <nav ng-if="job" class="navbar navbar-default navbar-fixed-top navbar-main-additional">
   <ul class="nav nav-tabs">

flink-runtime-web/web-dashboard/web/partials/jobs/job.plan.node.html

@@ -23,7 +23,7 @@ limitations under the License.
     <div class="panel-info first">ID: {{ node.id }}</div>
     <div ng-if="node.description" class="panel-info">
       <div class="label-group">
-        <bs-label status="{{status}}" ng-repeat="(index, status) in stateList">{{node.vertex.groupvertex[status]}}</bs-label>
+        <bs-label status="{{status}}" ng-repeat="(index, status) in stateList">{{node.vertex.tasks[status]}}</bs-label>
       </div>
     </div>
   </div>
@@ -40,7 +40,7 @@ limitations under the License.
       </thead>
       <tbody>
         <tr ng-repeat="vertex in node.vertex.groupvertex.groupmembers">
-          <td>{{ vertex.vertexname | humanizeTaskName }}</td>
+          <td>{{ vertex.name | humanizeTaskName }}</td>
           <td> 
             <bs-label status="{{vertex.vertexstatus}}">{{vertex.vertexstatus}}</bs-label>
           </td>

flink-runtime-web/web-dashboard/web/partials/jobs/running-jobs.html

@@ -26,8 +26,6 @@ limitations under the License.
     <thead>
       <tr>
         <th>Start Time</th>
-        <th>End Time</th>
-        <th>Duration</th>
         <th>Job Name</th>
         <th>Job ID</th>
         <th>Tasks</th>
@@ -37,8 +35,6 @@ limitations under the License.
     <tbody>
       <tr ng-repeat="job in jobs" ui-sref="single-job.plan({ jobid: job.jid })">
         <td>{{job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}</td>
-        <td>{{job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}</td>
-        <td>{{job['end-time'] - job['start-time']}} ms</td>
         <td>{{job.name}}</td>
         <td>{{job.jid}}</td>
         <td class="label-group">

flink-runtime-web/web-dashboard/web/partials/overview.html

@@ -83,8 +83,6 @@ limitations under the License.
         <thead>
           <tr>
             <th>Start Time</th>
-            <th>End Time</th>
-            <th>Duration</th>
             <th>Job Name</th>
             <th>Job ID</th>
             <th>Tasks</th>
@@ -94,8 +92,6 @@ limitations under the License.
         <tbody>
           <tr ng-repeat="job in runningJobs" ui-sref="single-job.plan({ jobid: job.jid })">
             <td>{{job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}</td>
-            <td>{{job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}</td>
-            <td>{{job.duration}}</td>
             <td>{{job.name}}</td>
             <td>{{job.jid}}</td>
             <td class="label-group">
@@ -130,7 +126,7 @@ limitations under the License.
           <tr ng-repeat="job in finishedJobs" ui-sref="single-job.plan({ jobid: job.jid })">
             <td>{{job['start-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}</td>
             <td>{{job['end-time'] | amDateFormat:'YYYY-MM-DD, H:mm:ss'}}</td>
-            <td>{{job['end-time'] - job['start-time']}} ms</td>
+            <td>{{job.duration}} ms</td>
             <td>{{job.name}}</td>
             <td>{{job.jid}}</td>
             <td class="label-group">