Skip to content

F
flink

doujutun3207 / flink
与 Fork 源项目一致

从无法访问的项目Fork

通知 24
Star 0
Fork 0
F
flink

体验新版 GitCode，发现更多精彩内容 >>

提交 85f061f2 编写于 作者: T Till Rohrmann
浏览文件
操作

Removed old java implementations of the JobManager, TaskManager, JobClient,... 

Removed old java implementations of the JobManager, TaskManager, JobClient, EventCollector, TaskOperationResult and MemoryArchivist.
上级 241c1ca4
展开全部 隐藏空白更改
内联 并排
Showing with 0 addition and 3178 deletion
flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClient2.old 已删除 100644 → 0
浏览文件 @ 241c1ca4
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.client;
import java.io.IOException;
import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.util.Iterator;
import java.util.Map;
import akka.actor.ActorPath;
import akka.actor.ActorRef;
import akka.actor.ActorRefProvider;
import akka.actor.ActorSystem;
import akka.actor.ActorSystemImpl;
import akka.actor.Extension;
import akka.actor.ExtensionId;
import akka.actor.InternalActorRef;
import akka.actor.Props;
import akka.actor.Scheduler;
import akka.dispatch.Dispatchers;
import akka.dispatch.Mailboxes;
import akka.event.EventStream;
import akka.event.LoggingAdapter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.accumulators.AccumulatorHelper;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.accumulators.AccumulatorEvent;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.event.job.AbstractEvent;
import org.apache.flink.runtime.event.job.JobEvent;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobmanager.JobManager;
import org.apache.flink.runtime.messages.EventCollectorMessages;
import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.runtime.messages.JobResult;
import org.apache.flink.runtime.messages.JobResult.JobCancelResult;
import org.apache.flink.runtime.messages.JobResult.JobSubmissionResult;
import org.apache.flink.runtime.messages.JobResult.JobProgressResult;
import org.apache.flink.util.StringUtils;
import scala.Function0;
import scala.concurrent.ExecutionContextExecutor;
import scala.concurrent.duration.Duration;
/**
* The job client is able to submit, control, and abort jobs.
*/
public class JobClient {
/** The logging object used for debugging. */
private static final Logger LOG = LoggerFactory.getLogger(JobClient.class);
private final ActorRef jobManager;
/**
private final JobManagementProtocol jobSubmitClient;
/** The job graph assigned with this job client. */
private final JobGraph jobGraph;
/**
* The configuration assigned with this job client.
*/
private final Configuration configuration;
private final ClassLoader userCodeClassLoader;
/**
* The sequence number of the last processed event received from the job manager.
*/
private long lastProcessedEventSequenceNumber = -1;
private PrintStream console;
/**
* Constructs a new job client object and instantiates a local
* RPC proxy for the JobSubmissionProtocol
*
* @param jobGraph
* the job graph to run
* @throws IOException
* thrown on error while initializing the RPC connection to the job manager
*/
public JobClient(JobGraph jobGraph, ClassLoader userCodeClassLoader) throws IOException {
this(jobGraph, new Configuration(), userCodeClassLoader);
}
/**
* Constructs a new job client object and instantiates a local
* RPC proxy for the JobSubmissionProtocol
*
* @param jobGraph
* the job graph to run
* @param configuration
* configuration object which can include special configuration settings for the job client
* @throws IOException
* thrown on error while initializing the RPC connection to the job manager
*/
public JobClient(JobGraph jobGraph, Configuration configuration, ClassLoader userCodeClassLoader) throws IOException {
final String address = configuration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
final int port = configuration.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY,
ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT);
final InetSocketAddress inetaddr = new InetSocketAddress(address, port);
ActorSystem system = ActorSystem.create("JobClientActorSystem",
AkkaUtils.getDefaultActorSystemConfig());
this.jobManager = JobManager.getJobManager(inetaddr, system);
this.jobGraph = jobGraph;
this.configuration = configuration;
this.userCodeClassLoader = userCodeClassLoader;
}
/**
* Returns the {@link Configuration} object which can include special configuration settings for the job client.
*
* @return the {@link Configuration} object which can include special configuration settings for the job client
*/
public Configuration getConfiguration() {
return this.configuration;
}
/**
* Submits the job assigned to this job client to the job manager.
*
* @return a <code>JobSubmissionResult</code> object encapsulating the results of the job submission
* @throws IOException
* thrown in case of submission errors while transmitting the data to the job manager
*/
public JobSubmissionResult submitJob() throws IOException {
// Get port of BLOB server
final int port = AkkaUtils.ask(jobManager, JobManagerMessages.RequestBlobManagerPort$
.MODULE$);
if (port == -1) {
throw new IOException("Unable to upload user jars: BLOB server not running");
}
// We submit the required files with the BLOB manager before the submission of the actual job graph
final String jobManagerAddress = configuration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
if(jobManagerAddress == null){
throw new IOException("Unable to find job manager address from configuration.");
}
final InetSocketAddress blobManagerAddress = new InetSocketAddress(jobManagerAddress,
port);
this.jobGraph.uploadRequiredJarFiles(blobManagerAddress);
try{
return AkkaUtils.ask(jobManager, new JobManagerMessages.SubmitJob(jobGraph));
}catch(IOException ioe) {
throw ioe;
}
}
/**
* Cancels the job assigned to this job client.
*
* @return a <code>JobCancelResult</code> object encapsulating the result of the job cancel request
* @throws IOException
* thrown if an error occurred while transmitting the request to the job manager
*/
public JobCancelResult cancelJob() throws IOException {
try{
return AkkaUtils.ask(jobManager, new JobManagerMessages.CancelJob(jobGraph.getJobID()));
}catch(IOException ioe){
throw ioe;
}
}
/**
* Retrieves the current status of the job assigned to this job client.
*
* @return a <code>JobProgressResult</code> object including the current job progress
* @throws IOException
* thrown if an error occurred while transmitting the request
*/
public JobProgressResult getJobProgress() throws IOException {
return AkkaUtils.ask(jobManager, new EventCollectorMessages.RequestJobProgress(jobGraph.getJobID()));
}
/**
* Submits the job assigned to this job client to the job manager and queries the job manager
* about the progress of the job until it is either finished or aborted.
*
* @return the duration of the job execution in milliseconds
* @throws IOException
* thrown if an error occurred while transmitting the request
* @throws JobExecutionException
* thrown if the job has been aborted either by the user or as a result of an error
*/
public JobExecutionResult submitJobAndWait() throws IOException, JobExecutionException {
final JobSubmissionResult submissionResult = submitJob();
if (submissionResult.returnCode() == JobResult.ERROR()) {
LOG.error("ERROR: " + submissionResult.description());
throw new JobExecutionException(submissionResult.description(), false);
}
long sleep = 0;
final int interval = AkkaUtils.<Integer>ask(jobManager, JobManagerMessages.RequestPollingInterval$
.MODULE$);
sleep = interval * 1000;
try {
Thread.sleep(sleep / 2);
} catch (InterruptedException e) {
logErrorAndRethrow(StringUtils.stringifyException(e));
}
long startTimestamp = -1;
while (true) {
if (Thread.interrupted()) {
logErrorAndRethrow("Job client has been interrupted");
}
JobResult.JobProgressResult jobProgressResult = null;
LOG.info("Request job progress.");
jobProgressResult = getJobProgress();
LOG.info("Requested job progress.");
if (jobProgressResult == null) {
logErrorAndRethrow("Returned job progress is unexpectedly null!");
}
if (jobProgressResult.returnCode() == JobResult.ERROR()) {
logErrorAndRethrow("Could not retrieve job progress: " + jobProgressResult.description());
}
final Iterator<AbstractEvent> it = jobProgressResult.asJavaList().iterator();
while (it.hasNext()) {
final AbstractEvent event = it.next();
// Did we already process that event?
if (this.lastProcessedEventSequenceNumber >= event.getSequenceNumber()) {
continue;
}
LOG.info(event.toString());
if (this.console != null) {
this.console.println(event.toString());
}
this.lastProcessedEventSequenceNumber = event.getSequenceNumber();
// Check if we can exit the loop
if (event instanceof JobEvent) {
final JobEvent jobEvent = (JobEvent) event;
final JobStatus jobStatus = jobEvent.getCurrentJobStatus();
if (jobStatus == JobStatus.RUNNING) {
startTimestamp = jobEvent.getTimestamp();
}
if (jobStatus == JobStatus.FINISHED) {
final long jobDuration = jobEvent.getTimestamp() - startTimestamp;
// Request accumulators
Map<String, Object> accumulators = null;
accumulators = AccumulatorHelper.toResultMap(getAccumulators().getAccumulators(this.userCodeClassLoader));
return new JobExecutionResult(jobDuration, accumulators);
} else if (jobStatus == JobStatus.CANCELED || jobStatus == JobStatus.FAILED) {
LOG.info(jobEvent.getOptionalMessage());
if (jobStatus == JobStatus.CANCELED) {
throw new JobExecutionException(jobEvent.getOptionalMessage(), true);
} else {
throw new JobExecutionException(jobEvent.getOptionalMessage(), false);
}
}
}
}
try {
Thread.sleep(sleep);
} catch (InterruptedException e) {
logErrorAndRethrow(StringUtils.stringifyException(e));
}
}
}
public int getRecommendedPollingInterval(){
try {
return AkkaUtils.<Integer>ask(jobManager, JobManagerMessages.RequestPollingInterval$.MODULE$);
}catch(IOException ioe){
throw new RuntimeException("Could not request recommended polling interval from job " +
"manager.", ioe);
}
}
/**
* Writes the given error message to the log and throws it in an {@link IOException}.
*
* @param errorMessage
* the error message to write to the log
* @throws IOException
* thrown after the error message is written to the log
*/
private void logErrorAndRethrow(final String errorMessage) throws IOException {
LOG.error(errorMessage);
throw new IOException(errorMessage);
}
public void setConsoleStreamForReporting(PrintStream stream) {
this.console = stream;
}
private AccumulatorEvent getAccumulators() throws IOException {
return AkkaUtils.ask(jobManager, new JobManagerMessages.RequestAccumulatorResult(jobGraph.getJobID()));
}
}
flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/EventCollector.old 已删除 100644 → 0
浏览文件 @ 241c1ca4
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.jobmanager;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import org.apache.flink.runtime.event.job.AbstractEvent;
import org.apache.flink.runtime.event.job.ExecutionStateChangeEvent;
import org.apache.flink.runtime.event.job.JobEvent;
import org.apache.flink.runtime.event.job.ManagementEvent;
import org.apache.flink.runtime.event.job.RecentJobEvent;
import org.apache.flink.runtime.event.job.VertexEvent;
import org.apache.flink.runtime.execution.ExecutionListener;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
import org.apache.flink.runtime.executiongraph.JobStatusListener;
import org.apache.flink.runtime.jobgraph.JobID;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobmanager.archive.ArchiveListener;
import org.apache.flink.runtime.profiling.ProfilingListener;
import org.apache.flink.runtime.profiling.types.ProfilingEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The event collector collects events which occurred during the execution of a job and prepares them
* for being fetched by a client. The collected events have an expiration time. In a configurable interval
* the event collector removes all intervals which are older than the interval.
*/
public final class EventCollector2 extends TimerTask implements ProfilingListener {
private static final Logger LOG = LoggerFactory.getLogger(EventCollector.class);
/**
* The execution listener wrapper is an auxiliary class. It is required
* because the job vertex ID and the management vertex ID cannot be accessed from
* the data provided by the <code>executionStateChanged</code> callback method.
* However, these IDs are needed to create the construct the {@link VertexEvent} and the
* {@link ExecutionStateChangeEvent}.
*/
private static final class ExecutionListenerWrapper implements ExecutionListener {
/** The event collector to forward the created event to. */
private final EventCollector eventCollector;
private final ExecutionGraph graph;
public ExecutionListenerWrapper(EventCollector eventCollector, ExecutionGraph graph) {
this.eventCollector = eventCollector;
this.graph = graph;
}
@Override
public void executionStateChanged(JobID jobID, JobVertexID vertexId, int subtask, ExecutionAttemptID executionId,
ExecutionState newExecutionState, String optionalMessage)
{
final long timestamp = System.currentTimeMillis();
final ExecutionJobVertex vertex = graph.getJobVertex(vertexId);
final String taskName = vertex == null ? "(null)" : vertex.getJobVertex().getName();
final int totalNumberOfSubtasks = vertex == null ? -1 : vertex.getParallelism();
// Create a new vertex event
final VertexEvent vertexEvent = new VertexEvent(timestamp, vertexId, taskName, totalNumberOfSubtasks,
subtask, executionId, newExecutionState, optionalMessage);
this.eventCollector2.addEvent(jobID, vertexEvent);
final ExecutionStateChangeEvent executionStateChangeEvent = new ExecutionStateChangeEvent(timestamp, vertexId, subtask,
executionId, newExecutionState);
this.eventCollector.addEvent(jobID, executionStateChangeEvent);
LOG.info(vertexEvent.toString());
}
}
/**
* The job status listener wrapper is an auxiliary class. It is required
* because the job name cannot be accessed from the data provided by the <code>jobStatusHasChanged</code> callback
* method. However, this job name
* is needed to create the construct the {@link RecentJobEvent}.
*
*/
private static final class JobStatusListenerWrapper implements JobStatusListener {
/** The event collector to forward the created event to. */
private final EventCollector eventCollector;
/** The name of the job this wrapper has been created for. */
private final String jobName;
/** <code>true</code> if profiling events are collected for the job, <code>false</code> otherwise. */
private final boolean isProfilingAvailable;
/** The time stamp of the job submission */
private final long submissionTimestamp;
/**
* Constructs a new job status listener wrapper.
*
* @param eventCollector2
* the event collector to forward the events to
* @param jobName
* the name of the job
* @param isProfilingAvailable
* <code>true</code> if profiling events are collected for the job, <code>false</code> otherwise
* @param submissionTimestamp
* the submission time stamp of the job
*/
public JobStatusListenerWrapper(EventCollector eventCollector, String jobName,
boolean isProfilingAvailable, long submissionTimestamp)
{
this.eventCollector = eventCollector;
this.jobName = jobName;
this.isProfilingAvailable = isProfilingAvailable;
this.submissionTimestamp = submissionTimestamp;
}
@Override
public void jobStatusHasChanged(ExecutionGraph executionGraph, JobStatus newJobStatus, String optionalMessage) {
final JobID jobID = executionGraph.getJobID();
if (newJobStatus == JobStatus.RUNNING) {
this.eventCollector.addExecutionGraph(jobID, executionGraph);
}
// Update recent job event
this.eventCollector.updateRecentJobEvent(jobID, this.jobName, this.isProfilingAvailable,
this.submissionTimestamp, newJobStatus);
this.eventCollector.addEvent(jobID,
new JobEvent(System.currentTimeMillis(), newJobStatus, optionalMessage));
}
}
private final long timerTaskInterval;
/**
* The map which stores all collected events until they are either
* fetched by the client or discarded.
*/
private final Map<JobID, List<AbstractEvent>> collectedEvents = new HashMap<JobID, List<AbstractEvent>>();
/**
* Map of recently started jobs with the time stamp of the last received job event.
*/
private final Map<JobID, RecentJobEvent> recentJobs = new HashMap<JobID, RecentJobEvent>();
/**
* Map of management graphs belonging to recently started jobs with the time stamp of the last received job event.
*/
private final Map<JobID, ExecutionGraph> recentManagementGraphs = new HashMap<JobID, ExecutionGraph>();
/**
* The timer used to trigger the cleanup routine.
*/
private final Timer timer;
private List<ArchiveListener> archivists = new ArrayList<ArchiveListener>();
/**
* Constructs a new event collector and starts
* its background cleanup routine.
*
* @param clientQueryInterval
* the interval with which clients query for events
*/
public EventCollector2(final int clientQueryInterval) {
this.timerTaskInterval = clientQueryInterval * 1000L * 2L; // Double the interval, clients will take care of
// duplicate notifications
this.timer = new Timer();
this.timer.schedule(this, this.timerTaskInterval, this.timerTaskInterval);
}
/**
* Retrieves and adds the collected events for the job with the given job ID to the provided list.
*
* @param jobID
* the ID of the job to retrieve the events for
* @param eventList
* the list to which the events shall be added
* @param includeManagementEvents
* <code>true</code> if {@link ManagementEvent} objects shall be added to the list as well,
* <code>false</code> otherwise
*/
public void getEventsForJob(JobID jobID, List<AbstractEvent> eventList, boolean includeManagementEvents) {
synchronized (this.collectedEvents) {
List<AbstractEvent> eventsForJob = this.collectedEvents.get(jobID);
if (eventsForJob != null) {
final Iterator<AbstractEvent> it = eventsForJob.iterator();
while (it.hasNext()) {
final AbstractEvent event = it.next();
final boolean isManagementEvent = (event instanceof ManagementEvent);
if (!isManagementEvent || includeManagementEvents) {
eventList.add(event);
}
}
}
}
}
public void getRecentJobs(List<RecentJobEvent> eventList) {
synchronized (this.recentJobs) {
eventList.addAll(this.recentJobs.values());
}
}
/**
* Stops the timer thread and cleans up the
* data structure which stores the collected events.
*/
public void shutdown() {
// Clear event map
synchronized (this.collectedEvents) {
this.collectedEvents.clear();
}
synchronized (this.recentJobs) {
this.recentJobs.clear();
}
// Cancel the timer for the cleanup routine
this.timer.cancel();
}
/**
* Adds an event to the job's event list.
*
* @param jobID
* the ID of the job the event belongs to
* @param event
* the event to be added to the job's event list
*/
private void addEvent(JobID jobID, AbstractEvent event) {
synchronized (this.collectedEvents) {
List<AbstractEvent> eventList = this.collectedEvents.get(jobID);
if (eventList == null) {
eventList = new ArrayList<AbstractEvent>();
this.collectedEvents.put(jobID, eventList);
}
eventList.add(event);
}
}
/**
* Creates a {@link RecentJobEvent} and adds it to the list of recent jobs.
*
* @param jobID
* the ID of the new job
* @param jobName
* the name of the new job
* @param isProfilingEnabled
* <code>true</code> if profiling events are collected for the job, <code>false</code> otherwise
* @param submissionTimestamp
* the submission time stamp of the job
* @param jobStatus
* the status of the job
*/
private void updateRecentJobEvent(JobID jobID, String jobName, boolean isProfilingEnabled,
long submissionTimestamp, JobStatus jobStatus)
{
final long currentTime = System.currentTimeMillis();
final RecentJobEvent recentJobEvent = new RecentJobEvent(jobID, jobName, jobStatus, isProfilingEnabled,
submissionTimestamp, currentTime);
synchronized (this.recentJobs) {
this.recentJobs.put(jobID, recentJobEvent);
}
}
/**
* Registers a job in form of its execution graph representation
* with the job progress collector. The collector will subscribe
* to state changes of the individual subtasks. A separate
* de-registration is not necessary since the job progress collector
* periodically discards outdated progress information.
*
* @param executionGraph
* the execution graph representing the job
* @param profilingAvailable
* indicates if profiling data is available for this job
* @param submissionTimestamp
* the submission time stamp of the job
*/
public void registerJob(ExecutionGraph executionGraph, boolean profilingAvailable, long submissionTimestamp) {
executionGraph.registerExecutionListener(new ExecutionListenerWrapper(this, executionGraph));
executionGraph.registerJobStatusListener(new JobStatusListenerWrapper(this, executionGraph.getJobName(),
profilingAvailable, submissionTimestamp));
}
/**
* This method will periodically be called to clean up expired
* collected events.
*/
@Override
public void run() {
final long currentTime = System.currentTimeMillis();
synchronized (this.collectedEvents) {
final Iterator<JobID> it = this.collectedEvents.keySet().iterator();
while (it.hasNext()) {
final JobID jobID = it.next();
final List<AbstractEvent> eventList = this.collectedEvents.get(jobID);
if (eventList == null) {
continue;
}
final Iterator<AbstractEvent> it2 = eventList.iterator();
while (it2.hasNext()) {
final AbstractEvent event = it2.next();
// If the event is older than TIMERTASKINTERVAL, remove it
if ((event.getTimestamp() + this.timerTaskInterval) < currentTime) {
archiveEvent(jobID, event);
it2.remove();
}
}
if (eventList.isEmpty()) {
it.remove();
}
}
}
synchronized (this.recentJobs) {
final Iterator<Map.Entry<JobID, RecentJobEvent>> it = this.recentJobs.entrySet().iterator();
while (it.hasNext()) {
final Map.Entry<JobID, RecentJobEvent> entry = it.next();
final JobStatus jobStatus = entry.getValue().getJobStatus();
// Only remove jobs from the list which have stopped running
if (jobStatus != JobStatus.FINISHED && jobStatus != JobStatus.CANCELED
&& jobStatus != JobStatus.FAILED) {
continue;
}
// Check time stamp of last job status update
if ((entry.getValue().getTimestamp() + this.timerTaskInterval) < currentTime) {
archiveJobevent(entry.getKey(), entry.getValue());
it.remove();
synchronized (this.recentManagementGraphs) {
archiveManagementGraph(entry.getKey(), this.recentManagementGraphs.get(entry.getKey()));
this.recentManagementGraphs.remove(entry.getValue());
}
}
}
}
}
@Override
public void processProfilingEvents(final ProfilingEvent profilingEvent) {
// Simply add profiling events to the job's event queue
addEvent(profilingEvent.getJobID(), profilingEvent);
}
/**
* Adds an execution graph to the map of recently created management graphs.
*
* @param jobID The ID of the graph
* @param executionGraph The graph to be added
*/
void addExecutionGraph(JobID jobID, ExecutionGraph executionGraph) {
synchronized (this.recentManagementGraphs) {
this.recentManagementGraphs.put(jobID, executionGraph);
}
}
/**
* Returns the execution graph object for the job with the given ID from the map of recently added graphs.
*
* @param jobID The ID of the job the management graph shall be retrieved for
* @return the management graph for the job with the given ID or <code>null</code> if no such graph exists
*/
public ExecutionGraph getManagementGraph(JobID jobID) {
synchronized (this.recentManagementGraphs) {
return this.recentManagementGraphs.get(jobID);
}
}
/**
* Register Archivist to archive
*/
public void registerArchivist(ArchiveListener al) {
this.archivists.add(al);
}
private void archiveEvent(JobID jobId, AbstractEvent event) {
for (ArchiveListener al : archivists) {
al.archiveEvent(jobId, event);
}
}
private void archiveJobevent(JobID jobId, RecentJobEvent event) {
for (ArchiveListener al : archivists) {
al.archiveJobevent(jobId, event);
}
}
private void archiveManagementGraph(JobID jobId, ExecutionGraph graph) {
for (ArchiveListener al : archivists) {
al.archiveExecutionGraph(jobId, graph);
}
}
}
flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/archive/MemoryArchivist2.old 已删除 100644 → 0
浏览文件 @ 241c1ca4
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.jobmanager.archive;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.apache.flink.runtime.event.job.AbstractEvent;
import org.apache.flink.runtime.event.job.RecentJobEvent;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
import org.apache.flink.runtime.jobgraph.JobID;
/**
* Implementation of the ArchiveListener, that archives old data of the JobManager in memory.
*
* This class must be thread safe, because it is accessed by the JobManager events and by the
* web server concurrently.
*/
public class MemoryArchivist2 implements ArchiveListener {
/** The global lock */
private final Object lock = new Object();
/**
* The map which stores all collected events until they are either
* fetched by the client or discarded.
*/
private final Map<JobID, List<AbstractEvent>> collectedEvents = new HashMap<JobID, List<AbstractEvent>>();
/** Map of recently started jobs with the time stamp of the last received job event. */
private final Map<JobID, RecentJobEvent> oldJobs = new HashMap<JobID, RecentJobEvent>();
/** Map of management graphs belonging to recently started jobs with the time stamp of the last received job event. */
private final Map<JobID, ExecutionGraph> graphs = new HashMap<JobID, ExecutionGraph>();
private final LinkedList<JobID> lru = new LinkedList<JobID>();
private final int max_entries;
// --------------------------------------------------------------------------------------------
public MemoryArchivist(int max_entries) {
this.max_entries = max_entries;
}
// --------------------------------------------------------------------------------------------
@Override
public void archiveExecutionGraph(JobID jobId, ExecutionGraph graph) {
synchronized (lock) {
graphs.put(jobId, graph);
cleanup(jobId);
}
}
@Override
public void archiveEvent(JobID jobId, AbstractEvent event) {
synchronized (lock) {
if(!collectedEvents.containsKey(jobId)) {
collectedEvents.put(jobId, new ArrayList<AbstractEvent>());
}
collectedEvents.get(jobId).add(event);
cleanup(jobId);
}
}
@Override
public void archiveJobevent(JobID jobId, RecentJobEvent event) {
synchronized (lock) {
oldJobs.put(jobId, event);
cleanup(jobId);
}
}
@Override
public List<RecentJobEvent> getJobs() {
synchronized (lock) {
return new ArrayList<RecentJobEvent>(oldJobs.values());
}
}
@Override
public RecentJobEvent getJob(JobID jobId) {
synchronized (lock) {
return oldJobs.get(jobId);
}
}
@Override
public List<AbstractEvent> getEvents(JobID jobID) {
synchronized (graphs) {
return collectedEvents.get(jobID);
}
}
@Override
public ExecutionGraph getExecutionGraph(JobID jid) {
synchronized (lock) {
return graphs.get(jid);
}
}
private void cleanup(JobID jobId) {
if (!lru.contains(jobId)) {
lru.addFirst(jobId);
}
if (lru.size() > this.max_entries) {
JobID toRemove = lru.removeLast();
collectedEvents.remove(toRemove);
oldJobs.remove(toRemove);
graphs.remove(toRemove);
}
}
}
flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskOperationResult.old 已删除 100644 → 0
浏览文件 @ 241c1ca4
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.taskmanager;
import java.io.IOException;
import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.util.StringUtils;
import com.google.common.base.Preconditions;
public class TaskOperationResult implements IOReadableWritable, java.io.Serializable {
private static final long serialVersionUID = -3852292420229699888L;
private ExecutionAttemptID executionId;
private boolean success;
private String description;
public TaskOperationResult() {
this(new ExecutionAttemptID(), false);
}
public TaskOperationResult(ExecutionAttemptID executionId, boolean success) {
this(executionId, success, null);
}
public TaskOperationResult(ExecutionAttemptID executionId, boolean success, String description) {
Preconditions.checkNotNull(executionId);
this.executionId = executionId;
this.success = success;
this.description = description;
}
public ExecutionAttemptID getExecutionId() {
return executionId;
}
public boolean isSuccess() {
return success;
}
public String getDescription() {
return description;
}
// --------------------------------------------------------------------------------------------
// Serialization
// --------------------------------------------------------------------------------------------
@Override
public void read(DataInputView in) throws IOException {
this.executionId.read(in);
this.success = in.readBoolean();
this.description = StringUtils.readNullableString(in);
}
@Override
public void write(DataOutputView out) throws IOException {
this.executionId.write(out);
out.writeBoolean(success);
StringUtils.writeNullableString(description, out);
}
// --------------------------------------------------------------------------------------------
// Utilities
// --------------------------------------------------------------------------------------------
@Override
public String toString() {
return String.format("TaskOperationResult %s [%s]%s", executionId,
success ? "SUCCESS" : "FAILED", description == null ? "" : " - " + description);
}
@Override
public boolean equals(Object o){
if(o == null){
return false;
}
if(o instanceof TaskOperationResult){
TaskOperationResult tor = (TaskOperationResult) o;
boolean result = true;
if(executionId == null){
if(tor.executionId != null){
return false;
}
}else{
result &= executionId.equals(tor.executionId);
}
result &= success == tor.success;
if(description == null){
if(tor.description != null){
return false;
}
}else{
result &= description.equals(tor.description);
}
return result;
}else{
return false;
}
}
}
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或