[yarn] fixed yarn binary; implemented a simple but effective shutdown...

[yarn] fixed yarn binary; implemented a simple but effective shutdown mechanism for the yarn-session

stratosphere-addons/yarn/src/main/java/eu/stratosphere/yarn/ApplicationMaster.java

@@ -19,7 +19,6 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileWriter;
-import java.io.FilenameFilter;
 import java.io.InputStreamReader;
 import java.io.Writer;
 import java.util.Collections;
@@ -163,8 +162,6 @@ public class ApplicationMaster {
 			LOG.info("Requesting TaskManager container " + i);
 			rmClient.addContainerRequest(containerAsk);
 		}
-
-		
 		
 		LocalResource stratosphereJar = Records.newRecord(LocalResource.class);
 		LocalResource stratosphereConf = Records.newRecord(LocalResource.class);

stratosphere-addons/yarn/src/main/java/eu/stratosphere/yarn/Client.java

@@ -294,7 +294,7 @@ public class Client {
 		
 		// Set-up ApplicationSubmissionContext for the application
 		ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
-		ApplicationId appId = appContext.getApplicationId();
+		final ApplicationId appId = appContext.getApplicationId();
 		
 		// Setup jar for ApplicationMaster
 		LocalResource appMasterJar = Records.newRecord(LocalResource.class);
@@ -337,9 +337,19 @@ public class Client {
 		LOG.info("Submitting application master " + appId);
 		yarnClient.submitApplication(appContext);
 		
+		
+		
 		 Runtime.getRuntime().addShutdownHook(new Thread() {
 		   @Override
 		   public void run() {
+		    try {
+		    	LOG.info("Killing the YARN instance.");
+				yarnClient.killApplication(appId);
+			} catch (YarnException e) {
+				e.printStackTrace();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
 		    LOG.info("YARN Client is shutting down");
 		    yarnClient.stop();
 		   }
@@ -393,9 +403,9 @@ public class Client {
 		formatter.setSyntaxPrefix("   Optional");
 		Options opt = new Options();
 		opt.addOption(VERBOSE);
-		opt.addOption(GEN_CONF);
-		opt.addOption(STRATOSPHERE_CONF);
-		opt.addOption(STRATOSPHERE_JAR);
+	//	opt.addOption(GEN_CONF);
+	//	opt.addOption(STRATOSPHERE_CONF);
+	//	opt.addOption(STRATOSPHERE_JAR);
 		opt.addOption(JM_MEMORY);
 		opt.addOption(TM_MEMORY);
 		opt.addOption(TM_CORES);

stratosphere-dist/pom.xml

@@ -121,6 +121,12 @@
 									<goal>single</goal>
 								</goals>
 								<configuration>
+					                <archiverConfig>
+					                	<!-- https://jira.codehaus.org/browse/MASSEMBLY-449 -->
+					                    <fileMode>420</fileMode> <!-- 420(dec) = 644(oct) -->
+								    	<directoryMode>493</directoryMode> <!-- 493(dec) = 755(oct) -->
+										<defaultDirectoryMode>493</defaultDirectoryMode>
+					                </archiverConfig>
 									<archive>
 										<manifest>
 											<mainClass>eu.stratosphere.yarn.Client</mainClass>

stratosphere-dist/src/main/assemblies/src.xml

-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
-    <id>src</id>
-    <formats>
-        <format>dir</format>
-
-		<!--
-        <format>tar.gz</format>
-        <format>zip</format>
-        -->
-    </formats>
-
-    <includeBaseDirectory>true</includeBaseDirectory>
-    <baseDirectory>stratosphere-${project.version}</baseDirectory>
-    
-    <!-- copy all jar files and their dependencies -->
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>src</outputDirectory>
-            <unpack>true</unpack>
-            <unpackOptions>
-              <excludes>
-                <exclude>META-INF/MANIFEST.MF</exclude>
-                <exclude>META-INF</exclude>
-              </excludes>
-            </unpackOptions>
-            <useTransitiveDependencies>false</useTransitiveDependencies>
-            <useProjectArtifact>false</useProjectArtifact>
-            <useProjectAttachments>false</useProjectAttachments>
-
-            <includes>
-              <include>*:sources</include>
-            </includes>
-        </dependencySet>
-        
-    </dependencySets>
-
-    <fileSets>
-    
-    	<fileSet>
-        <!-- copy start scripts -->
-    		<directory>src/main/stratosphere-bin/bin</directory>
-    		<outputDirectory>bin</outputDirectory>
-    		<fileMode>0755</fileMode>
-    	</fileSet>
-    	
-    	<fileSet>
-        <!-- copy default configuration -->
-    		<directory>src/main/stratosphere-bin/conf</directory>
-    		<outputDirectory>conf</outputDirectory>
-    		<fileMode>0644</fileMode>
-    	</fileSet>
-    	
-    	<fileSet>
-        <!-- copy README file -->
-    		<directory>src/main/stratosphere-bin/docs</directory>
-    		<outputDirectory>docs</outputDirectory>
-    		<fileMode>0644</fileMode>
-    	</fileSet>
-    	
-    	<fileSet>
-    	<!-- copy JavaDocs -->
-    		<!--
-    		<directory>../target/apidocs</directory>
-    		<directory>../target</directory>
-    		<includes>
-    			<include>stratosphere-*-javadoc.jar</include>
-    		</includes>
-    		<outputDirectory>docs/javadoc</outputDirectory>
-    		<fileMode>0644</fileMode>
-		-->
-    	</fileSet>
-    	
-	        <!-- copy files for PACT web frontend -->
-		<fileSet>
-			<directory>../pact/pact-clients/resources</directory>
-			<outputDirectory>resources</outputDirectory>
-			<fileMode>0644</fileMode>
-			<excludes>
-				<exclude>*etc/users</exclude>
-			</excludes>
-		</fileSet>
-
-		<!-- copy files for Jobmanager web frontend -->
-		<fileSet>
-			<directory>../nephele/nephele-server/resources</directory>
-			<outputDirectory>resources</outputDirectory>
-			<fileMode>0644</fileMode>
-			<excludes>
-				<exclude>*etc/users</exclude>
-			</excludes>
-		</fileSet>
-      
-    </fileSets>
-
-</assembly>

stratosphere-dist/src/main/assemblies/yarn-uberjar.xml

@@ -17,6 +17,9 @@
 			<useProjectArtifact>true</useProjectArtifact>
 			<unpack>true</unpack>
 			<scope>runtime</scope>
+			<excludes>
+				<exclude>eu.stratosphere:stratosphere-examples:*</exclude>
+			</excludes>
 		</dependencySet>
 	</dependencySets>
 

stratosphere-dist/src/main/assemblies/yarn.xml

@@ -2,7 +2,7 @@
 	xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
 	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
-	<id>bin</id>
+	<id>yarn</id>
 	<formats>
 		<format>tar.gz</format>
 		<!--
@@ -14,41 +14,7 @@
 	<includeBaseDirectory>true</includeBaseDirectory>
 	<baseDirectory>stratosphere-yarn-${project.version}</baseDirectory>
 
-	<files>
-		<!-- copy default configuration -->
-		<file>
-			<source>src/main/stratosphere-bin/conf/stratosphere-conf.yaml</source>
-			<outputDirectory>conf</outputDirectory>
-			<fileMode>0644</fileMode>
-		</file>
-		<file>
-			<source>src/main/stratosphere-bin/conf/log4j.properties</source>
-			<outputDirectory>conf</outputDirectory>
-			<fileMode>0644</fileMode>
-		</file>
-
-		<!-- copy uberjar -->
-		<file>
-			<source>target/stratosphere-dist-${project.version}-yarn-uberjar.jar</source>
-			<outputDirectory>lib/</outputDirectory>
-			<fileMode>0644</fileMode>
-		</file>
-
-		<!-- copy stratosphere client -->
-		<file>
-			<source>src/main/stratosphere-bin/bin/stratosphere</source>
-			<outputDirectory>bin/</outputDirectory>
-			<fileMode>0644</fileMode>
-		</file>
-		<file>
-			<source>src/main/stratosphere-bin/bin/config.sh</source>
-			<outputDirectory>bin/</outputDirectory>
-			<fileMode>0644</fileMode>
-		</file>
-	</files>
-
 	<fileSets>
-
 		<fileSet>
 			<!-- copy start scripts -->
 			<directory>src/main/stratosphere-bin/yarn-bin</directory>
@@ -110,4 +76,37 @@
 			</excludes>
 		</fileSet>
 	</fileSets>
+
+	<files>
+		<!-- copy default configuration -->
+		<file>
+			<source>src/main/stratosphere-bin/conf/stratosphere-conf.yaml</source>
+			<outputDirectory>conf</outputDirectory>
+			<fileMode>0644</fileMode>
+		</file>
+		<file>
+			<source>src/main/stratosphere-bin/conf/log4j.properties</source>
+			<outputDirectory>conf</outputDirectory>
+			<fileMode>0644</fileMode>
+		</file>
+
+		<!-- copy uberjar -->
+		<file>
+			<source>target/stratosphere-dist-${project.version}-yarn-uberjar.jar</source>
+			<outputDirectory>lib/</outputDirectory>
+			<fileMode>0644</fileMode>
+		</file>
+
+		<!-- copy stratosphere client -->
+		<file>
+			<source>src/main/stratosphere-bin/bin/stratosphere</source>
+			<outputDirectory>bin/</outputDirectory>
+			<fileMode>0644</fileMode>
+		</file>
+		<file>
+			<source>src/main/stratosphere-bin/bin/config.sh</source>
+			<outputDirectory>bin/</outputDirectory>
+			<fileMode>0755</fileMode>
+		</file>
+	</files>
 </assembly>

stratosphere-dist/src/main/stratosphere-bin/yarn-bin/yarn-session.sh

@@ -43,10 +43,11 @@ constructCLIClientClassPath() {
 
 CC_CLASSPATH=`manglePathList $(constructCLIClientClassPath)`
 
-log=$STRATOSPHERE_LOG_DIR/stratosphere-$STRATOSPHERE_IDENT_STRING-yarn-session-$HOSTNAME.log
-log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$STRATOSPHERE_CONF_DIR"/log4j.properties"
+#log=$STRATOSPHERE_LOG_DIR/stratosphere-$STRATOSPHERE_IDENT_STRING-yarn-session-$HOSTNAME.log
+#log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$STRATOSPHERE_CONF_DIR"/log4j.properties"
 
 export STRATOSPHERE_CONF_DIR
+# $log_setting
 
-$JAVA_RUN $JVM_ARGS $log_setting -classpath $CC_CLASSPATH eu.stratosphere.yarn.Client -j $STRATOSPHERE_LIB_DIR/*yarn-uberjar.jar -c $STRATOSPHERE_CONF_DIR/stratosphere-conf.yaml $*
+$JAVA_RUN $JVM_ARGS  -classpath $CC_CLASSPATH eu.stratosphere.yarn.Client -j $STRATOSPHERE_LIB_DIR/*yarn-uberjar.jar -c $STRATOSPHERE_CONF_DIR/stratosphere-conf.yaml $*