AppHost: Support bundles (stage 1) (dotnet/core-setup#5742)

* AppHost: Support bundles (stage 1)

This changes implements the [app-host support](https://github.com/dotnet/designs/blob/master/accepted/single-file/design.md#the-host )
for executing .net core apps published as a single file.

This change implements [stage 1](https://github.com/dotnet/designs/blob/master/accepted/single-file/staging.mddotnet/core-setup#1-self-extractor),
which [extracts out](https://github.com/dotnet/designs/blob/master/accepted/single-file/extract.md ) the embedded files to disk.

On startup, the AppHost detects whether it's own binary is a .net core bundle.
If so, on the first run, the host extracts embedded files to:
  * if `DOTNET_BUNDLE_EXTRACT_BASE_DIR` is set, to `$DOTNET_BUNDLE_EXTRACT_BASE_DIR/.net/<app>/<id>/...` . Otherwise, 
  * On Windows, to `%TEMP%/.net/<app>/<id>/...`
  * On Unix systems, if `$TMPDIR` is set, to `$TMPDIR/.net/<app>/<id>/...` . Otherwise to `/var/tmp` or `/tmp` if those paths are available and accessible.

On subsequent runs, the files extracted above are reused.

AppHost (X64) size increase:
Windows 5KB
Linux 11KB
Mac 6KB

Testing:
Tested the single-file extraction locally with several kinds of dotnet apps
(wpf, winforms, web, mvc, console, etc.)
Added a test case that runs a bundled apps with sub directories
Also fixed a bug in the Microsoft.NET.HostModel.extractor wrt processing bundled files in chunks.


Commit migrated from https://github.com/dotnet/core-setup/commit/4a01e55806b0849a08b820518e8cdcbeaaf07476

src/installer/corehost/cli/apphost/CMakeLists.txt

@@ -3,7 +3,7 @@
 # See the LICENSE file in the project root for more information.
 
 cmake_minimum_required (VERSION 2.6)
-project(apphost)
+project(apphost)	
 set(DOTNET_PROJECT_NAME "apphost")
 
 # Add RPATH to the apphost binary that allows using local copies of shared libraries
@@ -20,10 +20,17 @@ set(SKIP_VERSIONING 1)
 
 set(SOURCES
     ../fxr/fx_ver.cpp
+    ./bundle/file_entry.cpp
+    ./bundle/manifest.cpp
+    ./bundle/bundle_runner.cpp
 )
 
 set(HEADERS
     ../fxr/fx_ver.h
+    ./bundle/file_type.h
+    ./bundle/file_entry.h
+    ./bundle/manifest.h
+    ./bundle/bundle_runner.h
 )
 
 include(../exe.cmake)

src/installer/corehost/cli/apphost/bundle/bundle_runner.cpp

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include "bundle_runner.h"
+#include "pal.h"
+#include "trace.h"
+#include "utils.h"
+
+using namespace bundle;
+
+void bundle_runner_t::seek(FILE* stream, long offset, int origin)
+{
+    if (fseek(stream, offset, origin) != 0)
+    {
+        trace::error(_X("Failure processing application bundle; possible file corruption."));
+        trace::error(_X("I/O seek failure within the bundle."));
+        throw StatusCode::BundleExtractionIOError;
+    }
+}
+
+void bundle_runner_t::write(const void* buf, size_t size, FILE *stream)
+{
+    if (fwrite(buf, 1, size, stream) != size)
+    {
+        trace::error(_X("Failure extracting contents of the application bundle."));
+        trace::error(_X("I/O failure when writing extracted files."));
+        throw StatusCode::BundleExtractionIOError;
+    }
+}
+
+void bundle_runner_t::read(void* buf, size_t size, FILE* stream)
+{
+    if (fread(buf, 1, size, stream) != size)
+    {
+        trace::error(_X("Failure processing application bundle; possible file corruption."));
+        trace::error(_X("I/O failure reading contents of the bundle."));
+        throw StatusCode::BundleExtractionIOError;
+    }
+}
+
+// Read a non-null terminated fixed length UTF8 string from a byte-stream
+// and transform it to pal::string_t
+void bundle_runner_t::read_string(pal::string_t &str, size_t size, FILE* stream)
+{
+    uint8_t *buffer = new uint8_t[size + 1]; 
+    read(buffer, size, stream);
+    buffer[size] = 0; // null-terminator
+    pal::clr_palstring((const char*)buffer, &str);
+}
+
+static bool has_dirs_in_path(const pal::string_t& path)
+{
+    return path.find_last_of(DIR_SEPARATOR) != pal::string_t::npos;
+}
+
+static void create_directory_tree(const pal::string_t &path)
+{
+    if (path.empty())
+    {
+        return;
+    }
+
+    if (pal::directory_exists(path))
+    {
+        return;
+    }
+
+    if (has_dirs_in_path(path))
+    {
+        create_directory_tree(get_directory(path));
+    }
+
+    if (!pal::mkdir(path.c_str(), 0700))
+    {
+        if (pal::directory_exists(path))
+        {
+            // The directory was created since we last checked.
+            return;
+        }
+
+        trace::error(_X("Failure processing application bundle."));
+        trace::error(_X("Failed to create directory [%s] for extracting bundled files"), path.c_str());
+        throw StatusCode::BundleExtractionIOError;
+    }
+}
+
+static void remove_directory_tree(const pal::string_t& path)
+{
+    if (path.empty())
+    {
+        return;
+    }
+
+    std::vector<pal::string_t> dirs;
+    pal::readdir_onlydirectories(path, &dirs);
+
+    for (pal::string_t dir : dirs)
+    {
+        remove_directory_tree(dir);
+    }
+
+    std::vector<pal::string_t> files;
+    pal::readdir(path, &files);
+
+    for (pal::string_t file : files)
+    {
+        if (!pal::remove(file.c_str()))
+        {
+            trace::error(_X("Error removing file [%s]"), file.c_str());
+            throw StatusCode::BundleExtractionIOError;
+        }
+    }
+
+    if (!pal::rmdir(path.c_str()))
+    {
+        trace::error(_X("Error removing directory [%s]"), path.c_str());
+        throw StatusCode::BundleExtractionIOError;
+    }
+}
+
+void bundle_runner_t::reopen_host_for_reading()
+{
+    m_bundle_stream = pal::file_open(m_bundle_path, _X("rb"));
+    if (m_bundle_stream == nullptr)
+    {
+        trace::error(_X("Failure processing application bundle."));
+        trace::error(_X("Couldn't open host binary for reading contents"));
+        throw StatusCode::BundleExtractionIOError;
+    }
+}
+
+void bundle_runner_t::process_manifest_footer(int64_t &header_offset)
+{
+    seek(m_bundle_stream, -manifest_footer_t::num_bytes_read(), SEEK_END);
+
+    manifest_footer_t* footer = manifest_footer_t::read(m_bundle_stream);
+    header_offset = footer->manifest_header_offset();
+}
+
+void bundle_runner_t::process_manifest_header(int64_t header_offset)
+{
+    seek(m_bundle_stream, header_offset, SEEK_SET);
+
+    manifest_header_t* header = manifest_header_t::read(m_bundle_stream);
+
+    m_num_embedded_files = header->num_embedded_files();
+    m_bundle_id = header->bundle_id();
+}
+
+// Compute the final extraction location as:
+// m_extraction_dir = $DOTNET_BUNDLE_EXTRACT_BASE_DIR/<app>/<id>/...
+//
+// If DOTNET_BUNDLE_EXTRACT_BASE_DIR is not set in the environment, the 
+// base directory defaults to $TMPDIR/.net
+void bundle_runner_t::determine_extraction_dir()
+{
+    if (!pal::getenv(_X("DOTNET_BUNDLE_EXTRACT_BASE_DIR"), &m_extraction_dir))
+    {
+        if (!pal::get_temp_directory(m_extraction_dir))
+        {
+            trace::error(_X("Failure processing application bundle."));
+            trace::error(_X("Failed to determine location for extracting embedded files"));
+            throw StatusCode::BundleExtractionFailure;
+        }
+
+        append_path(&m_extraction_dir, _X(".net"));
+    }
+
+    pal::string_t host_name = strip_executable_ext(get_filename(m_bundle_path));
+    append_path(&m_extraction_dir, host_name.c_str());
+    append_path(&m_extraction_dir, m_bundle_id.c_str());
+
+    trace::info(_X("Files embedded within the bundled will be extracted to [%s] directory"), m_extraction_dir.c_str());
+}
+
+// Compute the worker extraction location for this process, before the 
+// extracted files are committed to the final location
+// m_working_extraction_dir = $DOTNET_BUNDLE_EXTRACT_BASE_DIR/<app>/<proc-id-hex>
+void bundle_runner_t::create_working_extraction_dir()
+{
+    // Set the working extraction path
+    m_working_extraction_dir = get_directory(m_extraction_dir);
+    pal::char_t pid[32];
+    pal::snwprintf(pid, 32, _X("%x"), pal::get_pid());
+    append_path(&m_working_extraction_dir, pid);
+
+    create_directory_tree(m_working_extraction_dir);
+
+    trace::info(_X("Temporary directory used to extract bundled files is [%s]"), m_working_extraction_dir.c_str());
+}
+
+// Create a file to be extracted out on disk, including any intermediate sub-directories.
+FILE* bundle_runner_t::create_extraction_file(const pal::string_t& relative_path)
+{
+    pal::string_t file_path = m_working_extraction_dir;
+    append_path(&file_path, relative_path.c_str());
+
+    // m_working_extraction_dir is assumed to exist, 
+    // so we only create sub-directories if relative_path contains directories
+    if (has_dirs_in_path(relative_path))
+    {
+        create_directory_tree(get_directory(file_path));
+    }
+
+    FILE* file = pal::file_open(file_path.c_str(), _X("wb"));
+
+    if (file == nullptr)
+    {
+        trace::error(_X("Failure processing application bundle."));
+        trace::error(_X("Failed to open file [%s] for writing"), file_path.c_str());
+        throw StatusCode::BundleExtractionIOError;
+    }
+
+    return file;
+}
+
+// Extract one file from the bundle to disk.
+void bundle_runner_t::extract_file(file_entry_t *entry)
+{
+    FILE* file = create_extraction_file(entry->relative_path);
+    const size_t buffer_size = 8 * 1024; // Copy the file in 8KB chunks
+    uint8_t buffer[buffer_size];
+    int64_t file_size = entry->data.size;
+
+    seek(m_bundle_stream, entry->data.offset, SEEK_SET);
+    do {
+        int64_t copy_size = (file_size <= buffer_size) ? file_size : buffer_size;
+        read(buffer, copy_size, m_bundle_stream);
+        write(buffer, copy_size, file);
+        file_size -= copy_size;
+    } while (file_size > 0);
+
+    fclose(file);
+}
+
+bool bundle_runner_t::can_reuse_extraction()
+{
+    // In this version, the extracted files are assumed to be 
+    // correct by construction.
+    // 
+    // Files embedded in the bundle are first extracted to m_working_extraction_dir
+    // Once all files are successfully extracted, the extraction location is 
+    // committed (renamed) to m_extraction_dir. Therefore, the presence of 
+    // m_extraction_dir means that the files are pre-extracted. 
+
+
+    return pal::directory_exists(m_extraction_dir);
+}
+
+// Current support for executing single-file bundles involves 
+// extraction of embedded files to actual files on disk. 
+// This method implements the file extraction functionality at startup.
+StatusCode bundle_runner_t::extract()
+{
+    try
+    {
+        // Determine if the current executable is a bundle
+        reopen_host_for_reading();
+
+        //  If the current AppHost is a bundle, it's layout will be 
+        //    AppHost binary 
+        //    Embedded Files: including the app, its configuration files, 
+        //                    dependencies, and possibly the runtime.
+        //    Bundle Manifest
+
+        int64_t manifest_header_offset;
+        process_manifest_footer(manifest_header_offset);
+        process_manifest_header(manifest_header_offset);
+
+        // Determine if embedded files are already extracted, and available for reuse
+        determine_extraction_dir();
+        if (can_reuse_extraction())
+        {
+            return StatusCode::Success;
+        }
+
+        // Extract files to temporary working directory
+        //
+        // Files are extracted to a specific deterministic location on disk
+        // on first run, and are available for reuse by subsequent similar runs.
+        //
+        // The extraction should be fault tolerant with respect to:
+        //  * Failures/crashes during extraction which result in partial-extraction
+        //  * Race between two or more processes concurrently attempting extraction
+        //
+        // In order to solve these issues, we implement a extraction as a two-phase approach:
+        // 1) Files embedded in a bundle are extracted to a process-specific temporary
+        //    extraction location (m_working_extraction_dir)
+        // 2) Upon successful extraction, m_working_extraction_dir is renamed to the actual
+        //    extraction location (m_extraction_dir)
+        //    
+        // This effectively creates a file-lock to protect against races and failed extractions.
+        
+        create_working_extraction_dir();
+
+        m_manifest = manifest_t::read(m_bundle_stream, m_num_embedded_files);
+
+        for (file_entry_t* entry : m_manifest->files) {
+            extract_file(entry);
+        }
+
+        // Commit files to the final extraction directory
+        if (pal::rename(m_working_extraction_dir.c_str(), m_extraction_dir.c_str()) != 0)
+        {
+            if (can_reuse_extraction())
+            {
+                // Another process successfully extracted the dependencies
+
+                trace::info(_X("Extraction completed by another process, aborting current extracion."));
+
+                remove_directory_tree(m_working_extraction_dir);
+                return StatusCode::Success;
+            }
+
+            trace::error(_X("Failure processing application bundle."));
+            trace::error(_X("Failed to commit extracted to files to directory [%s]"), m_extraction_dir.c_str());
+            throw StatusCode::BundleExtractionFailure;
+        }
+
+        fclose(m_bundle_stream);
+        return StatusCode::Success;
+    }
+    catch (StatusCode e)
+    {
+        fclose(m_bundle_stream);
+        return e;
+    }
+}

src/installer/corehost/cli/apphost/bundle/bundle_runner.h

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#ifndef __BUNDLE_RUNNER_H__
+#define __BUNDLE_RUNNER_H__
+
+
+#include <cstdint>
+#include "manifest.h"
+#include "error_codes.h"
+
+namespace bundle
+{
+    class bundle_runner_t
+    {
+    public:
+        bundle_runner_t(const pal::string_t& bundle_path)
+            :m_bundle_path(bundle_path),
+            m_bundle_stream(nullptr),
+            m_manifest(nullptr),
+            m_num_embedded_files(0)
+        {
+        }
+
+        pal::string_t get_extraction_dir()
+        {
+            return m_extraction_dir;
+        }
+
+        StatusCode extract();
+
+        static void read(void* buf, size_t size, FILE* stream);
+        static void write(const void* buf, size_t size, FILE* stream);
+        static void read_string(pal::string_t& str, size_t size, FILE* stream);
+
+    private:
+        void reopen_host_for_reading();
+        static void seek(FILE* stream, long offset, int origin);
+
+        void process_manifest_footer(int64_t& header_offset);
+        void process_manifest_header(int64_t header_offset);
+
+        void determine_extraction_dir();
+        void create_working_extraction_dir();
+        bool can_reuse_extraction();
+
+        FILE* create_extraction_file(const pal::string_t& relative_path);
+        void extract_file(file_entry_t* entry);
+
+        FILE* m_bundle_stream;
+        manifest_t* m_manifest;
+        int32_t m_num_embedded_files;
+        pal::string_t m_bundle_path;
+        pal::string_t m_bundle_id;
+        pal::string_t m_extraction_dir;
+        pal::string_t m_working_extraction_dir;
+    };
+
+}
+
+#endif // __BUNDLE_RUNNER_H__

src/installer/corehost/cli/apphost/bundle/file_entry.cpp

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include "bundle_runner.h"
+#include "pal.h"
+#include "error_codes.h"
+#include "trace.h"
+#include "utils.h"
+
+using namespace bundle;
+
+bool file_entry_t::is_valid()
+{
+    return data.offset > 0 && data.size > 0 &&
+        (file_type_t)data.type < file_type_t::__last &&
+        data.path_length > 0 && data.path_length <= PATH_MAX;
+}
+
+file_entry_t* file_entry_t::read(FILE* stream)
+{
+    file_entry_t* entry = new file_entry_t();
+
+    // First read the fixed-sized portion of file-entry
+    bundle_runner_t::read(&entry->data, sizeof(entry->data), stream);
+    if (!entry->is_valid())
+    {
+        trace::error(_X("Failure processing application bundle; possible file corruption."));
+        trace::error(_X("Invalid FileEntry detected."));
+        throw StatusCode::BundleExtractionFailure;
+    }
+
+    // Read the relative-path, given its length 
+    pal::string_t& path = entry->relative_path;
+    bundle_runner_t::read_string(path, entry->data.path_length, stream);
+
+    // Fixup the relative-path to have current platform's directory separator.
+    if (bundle_dir_separator != DIR_SEPARATOR)
+    {
+        for (size_t pos = path.find(bundle_dir_separator);
+            pos != pal::string_t::npos;
+            pos = path.find(bundle_dir_separator, pos))
+        {
+            path[pos] = DIR_SEPARATOR;
+        }
+    }
+
+    return entry;
+}
+
+

src/installer/corehost/cli/apphost/bundle/file_entry.h

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#ifndef __FILE_ENTRY_H__
+#define __FILE_ENTRY_H__
+
+#include <cstdint>
+#include "file_type.h"
+#include "pal.h"
+
+namespace bundle
+{
+
+    // FileEntry: Records information about embedded files.
+    // 
+    // The bundle manifest records the following meta-data for each 
+    // file embedded in the bundle:
+    // Fixed size portion (represented by file_entry_inner_t)
+    //   - Offset     
+    //   - Size       
+    //   - File Entry Type       
+    //   - path-length  (7-bit extension encoding, 1 Byte due to MAX_PATH)
+    // Variable Size portion
+    //   - relative path  ("path-length" Bytes)
+
+    class file_entry_t
+    {
+    public:
+
+        // The inner structure represents the fields that can be 
+        // read contiguously for every file_entry. 
+#pragma pack(push, 1)
+        struct
+        {
+            int64_t offset;
+            int64_t size;
+            file_type_t type;
+            int8_t path_length;
+        } data;
+#pragma pack(pop)
+        pal::string_t relative_path; // Path of an embedded file, relative to the extraction directory.
+
+        file_entry_t()
+            :data(), relative_path()
+        {
+        }
+
+        static file_entry_t* read(FILE* stream);
+
+    private:
+        static const pal::char_t bundle_dir_separator = '/';
+        bool is_valid();
+    };
+}
+#endif // __FILE_ENTRY_H__

src/installer/corehost/cli/apphost/bundle/file_type.h

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#ifndef __FILE_TYPE_H__
+#define __FILE_TYPE_H__
+
+#include <cstdint>
+
+namespace bundle
+{
+    // FileType: Identifies the type of file embedded into the bundle.
+    // 
+    // The bundler differentiates a few kinds of files via the manifest,
+    // with respect to the way in which they'll be used by the runtime.
+    //
+    // Currently all files are extracted out to the disk, but future 
+    // implementations will process certain file_types directly from the bundle.
+
+    enum file_type_t : uint8_t
+    {
+        assembly,
+        ready2run,
+        deps_json,
+        runtime_config_json,
+        extract,
+        __last
+    };
+}
+
+#endif // __FILE_TYPE_H__

src/installer/corehost/cli/apphost/bundle/manifest.cpp

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include "bundle_runner.h"
+#include "pal.h"
+#include "error_codes.h"
+#include "trace.h"
+#include "utils.h"
+
+using namespace bundle;
+
+bool manifest_header_t::is_valid()
+{
+    return m_data.major_version == m_current_major_version &&
+           m_data.minor_version == m_current_minor_version &&
+           m_data.num_embedded_files > 0 &&
+           m_data.bundle_id_length > 0 && 
+           m_data.bundle_id_length < PATH_MAX;
+}
+
+manifest_header_t* manifest_header_t::read(FILE* stream)
+{
+    manifest_header_t* header = new manifest_header_t();
+
+    // First read the fixed size portion of the header
+    bundle_runner_t::read(&header->m_data, sizeof(header->m_data), stream);
+    if (!header->is_valid())
+    {
+        trace::error(_X("Failure processing application bundle."));
+        trace::error(_X("Manifest header version compatibility check failed"));
+
+        throw StatusCode::BundleExtractionFailure;
+    }
+     
+    // Next read the bundle-ID string, given its length
+    bundle_runner_t::read_string(header->m_bundle_id, 
+                                 header->m_data.bundle_id_length, stream);
+
+    return header;
+}
+
+const char* manifest_footer_t::m_expected_signature = ".NetCoreBundle";
+
+bool manifest_footer_t::is_valid()
+{
+    return m_header_offset > 0 &&
+        m_signature_length == 14 &&
+        strcmp(m_signature, m_expected_signature) == 0;
+}
+
+manifest_footer_t* manifest_footer_t::read(FILE* stream)
+{
+    manifest_footer_t* footer = new manifest_footer_t();
+
+    bundle_runner_t::read(footer, num_bytes_read(), stream);
+
+    if (!footer->is_valid())
+    {
+        trace::info(_X("This executable is not recognized as a bundle."));
+
+        throw StatusCode::AppHostExeNotBundle;
+    }
+
+    return footer;
+}
+
+manifest_t* manifest_t::read(FILE* stream, int32_t num_files)
+{
+    manifest_t* manifest = new manifest_t();
+
+    for (int32_t i = 0; i < num_files; i++)
+    {
+        file_entry_t* entry = file_entry_t::read(stream);
+        if (entry == nullptr)
+        {
+            return nullptr;
+        }
+
+        manifest->files.push_back(entry);
+    }
+
+    return manifest;
+}

src/installer/corehost/cli/apphost/bundle/manifest.h

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#ifndef __MANIFEST_H__
+#define __MANIFEST_H__
+
+#include <cstdint>
+#include <list>
+#include "file_entry.h"
+
+namespace bundle
+{
+    // Manifest Header contains:
+    // Fixed size thunk (represened by manifest_header_inner_t)
+    //   - Major Version     
+    //   - Minor Version     
+    //   - Number of embedded files
+    //   - Bundle ID length 
+    // Variable size portion:
+    //   - Bundle ID ("Bundle ID length" bytes)
+
+    struct manifest_header_t
+    {
+    public:
+        manifest_header_t()
+            :m_data(), m_bundle_id()
+        {
+        }
+
+        bool is_valid();
+        static manifest_header_t* read(FILE* stream);
+        const pal::string_t& bundle_id() { return m_bundle_id; }
+        int32_t num_embedded_files() { return m_data.num_embedded_files;  }
+
+    private:
+#pragma pack(push, 1)
+        struct
+        {
+            uint32_t major_version;
+            uint32_t minor_version;
+            int32_t num_embedded_files;
+            int8_t bundle_id_length;
+        } m_data;
+#pragma pack(pop)
+        pal::string_t m_bundle_id;
+
+        static const uint32_t m_current_major_version = 0;
+        static const uint32_t m_current_minor_version = 1;
+    };
+
+    // Manifest Footer contains:
+    //   Manifest header offset
+    //   Length-prefixed non-null terminated Bundle Signature ".NetCoreBundle"
+#pragma pack(push, 1)
+    struct manifest_footer_t
+    {
+        manifest_footer_t()
+            :m_header_offset(0), m_signature_length(0)
+        {
+            // The signature string is not null-terminated as read from disk.
+            // We add an additional character for null termination
+            m_signature[14] = 0;
+        }
+
+        bool is_valid();
+        static manifest_footer_t* read(FILE* stream);
+        int64_t manifest_header_offset() { return m_header_offset; }
+        static size_t num_bytes_read()
+        {
+            return sizeof(manifest_footer_t) - 1;
+        }
+
+    private:
+        int64_t m_header_offset;
+        uint8_t m_signature_length;
+        char m_signature[15];
+
+    private:
+
+        static const char* m_expected_signature;
+    };
+#pragma pack(pop)
+
+
+    // Bundle Manifest contains:
+    //     Series of file entries (for each embedded file)
+
+    class manifest_t
+    {
+    public:
+        manifest_t()
+            :files()
+        {}
+
+        std::list<file_entry_t*> files;
+
+        static manifest_t* read(FILE* host, int32_t num_files);
+    };
+}
+#endif // __MANIFEST_H__

src/installer/corehost/common/pal.h

@@ -35,7 +35,10 @@
 #else
 
 #include <cstdlib>
+#include <unistd.h>
 #include <libgen.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 
 #define xerr std::cerr
 #define xout std::cout
@@ -136,6 +139,12 @@ namespace pal
     bool pal_clrstring(const pal::string_t& str, std::vector<char>* out);
     bool clr_palstring(const char* cstr, pal::string_t* out);
 
+    inline bool mkdir(const pal::char_t* dir, int mode) { return CreateDirectoryW(dir, NULL) != 0; }
+    inline bool rmdir (const pal::char_t* path) { return RemoveDirectoryW(path) != 0; }
+    inline int rename(const pal::char_t* old_name, const pal::char_t* new_name) { return ::_wrename(old_name, new_name); }
+    inline int remove(const pal::char_t* path) { return ::_wremove(path); }
+    inline int get_pid() { return GetCurrentProcessId(); }
+
 #else
     #ifdef EXPORT_SHARED_API
         #define SHARED_API extern "C" __attribute__((__visibility__("default")))
@@ -184,8 +193,23 @@ namespace pal
     inline bool pal_clrstring(const pal::string_t& str, std::vector<char>* out) { return pal_utf8string(str, out); }
     inline bool clr_palstring(const char* cstr, pal::string_t* out) { out->assign(cstr); return true; }
 
+    inline bool mkdir(const pal::char_t* dir, int mode) { return ::mkdir(dir, mode) == 0; }
+    inline bool rmdir(const pal::char_t* path) { return ::rmdir(path) == 0; }
+    inline int rename(const pal::char_t* old_name, const pal::char_t* new_name) { return ::rename(old_name, new_name); }
+    inline int remove(const pal::char_t* path) { return ::remove(path); }
+    inline int get_pid() { return getpid(); }
+
 #endif
 
+    inline int snwprintf(char_t* buffer, size_t count, const char_t* format, ...)
+    {
+        va_list args;
+        va_start(args, format);
+        int ret = str_vprintf(buffer, count, format, args);
+        va_end(args);
+        return ret;
+    }
+
     pal::string_t to_string(int value);
     pal::string_t get_timestamp();
 
@@ -230,6 +254,8 @@ namespace pal
     bool get_default_breadcrumb_store(string_t* recv);
     bool is_path_rooted(const string_t& path);
 
+    bool get_temp_directory(pal::string_t& tmp_dir);
+
     int xtoi(const char_t* input);
 
     bool load_library(const string_t* path, dll_t* dll);

src/installer/corehost/common/pal.unix.cpp

@@ -9,10 +9,7 @@
 #include <cassert>
 #include <dlfcn.h>
 #include <dirent.h>
-#include <sys/stat.h>
-#include <sys/types.h>
 #include <pwd.h>
-#include <unistd.h>
 #include <fcntl.h>
 #include <fnmatch.h>
 #include <ctime>
@@ -183,6 +180,34 @@ bool pal::get_default_servicing_directory(string_t* recv)
     return true;
 }
 
+bool pal::get_temp_directory(pal::string_t& tmp_dir)
+{
+    // First, check for the POSIX standard environment variable
+    if (pal::getenv(_X("TMPDIR"), &tmp_dir))
+    {
+        return pal::realpath(&tmp_dir);
+    }
+
+    // On non-compliant systems (ex: Ubuntu) try /var/tmp or /tmp directories.
+    // /var/tmp is prefered since its contents are expected to survive across
+    // machine reboot.
+    pal::string_t _var_tmp = _X("/var/tmp/");
+    if (pal::realpath(&_var_tmp))
+    {
+        tmp_dir.assign(_var_tmp);
+        return true;
+    }
+
+    pal::string_t _tmp = _X("/tmp/");
+    if (pal::realpath(&_tmp))
+    {
+        tmp_dir.assign(_tmp);
+        return true;
+    }
+
+    return false;
+}
+
 bool pal::get_global_dotnet_dirs(std::vector<pal::string_t>* recv)
 {
     // No support for global directories in Unix.

src/installer/corehost/common/pal.windows.cpp

@@ -457,6 +457,23 @@ bool pal::get_module_path(dll_t mod, string_t* recv)
     return GetModuleFileNameWrapper(mod, recv);
 }
 
+bool pal::get_temp_directory(pal::string_t& tmp_dir)
+{
+    const size_t max_len = MAX_PATH + 1;
+    pal::char_t temp_path[max_len];
+
+    size_t len = GetTempPathW(max_len, temp_path);
+    if (len == 0)
+    {
+        return false;
+    }
+
+    assert(len < max_len);
+    tmp_dir.assign(temp_path);
+
+    return pal::realpath(&tmp_dir);
+}
+
 static bool wchar_convert_helper(DWORD code_page, const char* cstr, int len, pal::string_t* out)
 {
     out->clear();

src/installer/corehost/common/utils.cpp

@@ -148,7 +148,7 @@ pal::string_t get_directory(const pal::string_t& path)
     {
         pos--;
     }
-    return ret.substr(0, pos + 1) + DIR_SEPARATOR;
+    return ret.substr(0, (size_t)pos + 1) + DIR_SEPARATOR;
 }
 
 void remove_trailing_dir_seperator(pal::string_t* dir)
@@ -161,7 +161,7 @@ void remove_trailing_dir_seperator(pal::string_t* dir)
 
 void replace_char(pal::string_t* path, pal::char_t match, pal::char_t repl)
 {
-    int pos = 0;
+	int pos = 0;
     while ((pos = path->find(match, pos)) != pal::string_t::npos)
     {
         (*path)[pos] = repl;
@@ -170,7 +170,7 @@ void replace_char(pal::string_t* path, pal::char_t match, pal::char_t repl)
 
 pal::string_t get_replaced_char(const pal::string_t& path, pal::char_t match, pal::char_t repl)
 {
-    int pos = path.find(match);
+	int pos = path.find(match);
     if (pos == pal::string_t::npos)
     {
         return path;

src/installer/corehost/corehost.cpp

@@ -11,6 +11,8 @@
 #include "utils.h"
 
 #if FEATURE_APPHOST
+#include "cli/apphost/bundle/bundle_runner.h"
+
 #define CURHOST_TYPE    _X("apphost")
 #define CUREXE_PKG_VER  COMMON_HOST_PKG_VER
 #define CURHOST_EXE
@@ -33,6 +35,7 @@
 #define EMBED_HASH_HI_PART_UTF8 "c3ab8ff13720e8ad9047dd39466b3c89" // SHA-256 of "foobar" in UTF-8
 #define EMBED_HASH_LO_PART_UTF8 "74e592c2fa383d4a3960714caef0c4f2"
 #define EMBED_HASH_FULL_UTF8    (EMBED_HASH_HI_PART_UTF8 EMBED_HASH_LO_PART_UTF8) // NUL terminated
+
 bool is_exe_enabled_for_execution(pal::string_t* app_dll)
 {
     constexpr int EMBED_SZ = sizeof(EMBED_HASH_FULL_UTF8) / sizeof(EMBED_HASH_FULL_UTF8[0]);
@@ -89,7 +92,7 @@ int exe_start(const int argc, const pal::char_t* argv[])
     pal::string_t app_path;
     pal::string_t app_root;
     bool requires_v2_hostfxr_interface = false;
-
+    
 #if FEATURE_APPHOST
     pal::string_t embedded_app_name;
     if (!is_exe_enabled_for_execution(&embedded_app_name))
@@ -109,7 +112,25 @@ int exe_start(const int argc, const pal::char_t* argv[])
         requires_v2_hostfxr_interface = true;
     }
 
-    app_path.assign(get_directory(host_path));
+    bundle::bundle_runner_t extractor(host_path);
+    StatusCode bundle_status = extractor.extract();
+
+    switch (bundle_status)
+    {
+    case StatusCode::Success:
+        app_path.assign(extractor.get_extraction_dir());
+        break;
+
+    case StatusCode::AppHostExeNotBundle:
+        app_path.assign(get_directory(host_path));
+        break;
+
+    case StatusCode::BundleExtractionFailure:
+    default:
+        trace::error(_X("A fatal error was encountered. Could not extract contents of the bundle"));
+        return StatusCode::AppHostExeNotBoundFailure;
+    }
+
     append_path(&app_path, embedded_app_name.c_str());
     if (!pal::realpath(&app_path))
     {
@@ -118,6 +139,7 @@ int exe_start(const int argc, const pal::char_t* argv[])
     }
 
     app_root.assign(get_directory(app_path));
+
 #else
     pal::string_t own_name = strip_executable_ext(get_filename(host_path));
 

src/installer/corehost/error_codes.h

@@ -36,5 +36,8 @@ enum StatusCode
     SdkResolverResolveFailure   = 0x8000809b,
     FrameworkCompatFailure      = 0x8000809c,
     FrameworkCompatRetry        = 0x8000809d,
+	AppHostExeNotBundle         = 0x8000809e,
+	BundleExtractionFailure     = 0x8000809f,
+	BundleExtractionIOError     = 0x800080a0
 };
 #endif // __ERROR_CODES_H__

src/installer/managed/Microsoft.NET.HostModel/Bundle/Extractor.cs

@@ -64,7 +64,7 @@ public void ExtractFiles()
                             long size = entry.Size;
                             do
                             {
-                                int copySize = (int)(size % int.MaxValue);
+                                int copySize = (int)(size <= int.MaxValue ? size : int.MaxValue);
                                 file.Write(reader.ReadBytes(copySize));
                                 size -= copySize;
                             } while (size > 0);

src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs

@@ -20,10 +20,10 @@ namespace Microsoft.NET.HostModel.Bundle
     /// </summary>
     public class FileEntry
     {
-        public FileType Type;
-        public string RelativePath; // Path of an embedded file, relative to the <app> dll.
         public long Offset;
         public long Size;
+        public FileType Type;
+        public string RelativePath; // Path of an embedded file, relative to the Bundle source-directory.
 
         public FileEntry(FileType fileType, string relativePath, long offset, long size)
         {
@@ -35,18 +35,18 @@ public FileEntry(FileType fileType, string relativePath, long offset, long size)
 
         public void Write(BinaryWriter writer)
         {
-            writer.Write((byte) Type);
-            writer.Write(RelativePath);
             writer.Write(Offset);
             writer.Write(Size);
+            writer.Write((byte)Type);
+            writer.Write(RelativePath);
         }
 
         public static FileEntry Read(BinaryReader reader)
         {
-            FileType type = (FileType)reader.ReadByte();
-            string fileName = reader.ReadString();
             long offset = reader.ReadInt64();
             long size = reader.ReadInt64();
+            FileType type = (FileType)reader.ReadByte();
+            string fileName = reader.ReadString();
             return new FileEntry(type, fileName, offset, size);
         }
 

src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs

@@ -32,6 +32,7 @@ namespace Microsoft.NET.HostModel.Bundle
     ///     MajorVersion
     ///     MinorVersion
     ///     NumEmbeddedFiles
+    ///     ExtractionID
     ///     
     /// - - - - - - Manifest Entries - - - - - - - - - - -
     ///     Series of FileEntries (for each embedded file)
@@ -51,11 +52,18 @@ public class Manifest
         public const uint MinorVersion = 1;
         public const char DirectorySeparatorChar = '/';
 
+        // Bundle ID is a string that is used to uniquely 
+        // identify this bundle. It is choosen to be compatible
+        // with path-names so that the AppHost can use it in
+        // extraction path.
+        string BundleID;
+
         public List<FileEntry> Files;
 
         public Manifest()
         {
             Files = new List<FileEntry>();
+            BundleID = Path.GetRandomFileName();
         }
 
         public long Write(BinaryWriter writer)
@@ -66,6 +74,7 @@ public long Write(BinaryWriter writer)
             writer.Write(MajorVersion);
             writer.Write(MinorVersion);
             writer.Write(Files.Count());
+            writer.Write(BundleID);
 
             // Write the manifest entries
             foreach (FileEntry entry in Files)
@@ -103,14 +112,14 @@ public static Manifest Read(BinaryReader reader)
             reader.BaseStream.Position = headerOffset;
             uint majorVersion = reader.ReadUInt32();
             uint minorVersion = reader.ReadUInt32();
+            int fileCount = reader.ReadInt32();
+            manifest.BundleID = reader.ReadString(); // Bundle ID
 
             if (majorVersion != MajorVersion || minorVersion != MinorVersion)
             {
                 throw new BundleException("Extraction failed: Invalid Version");
             }
 
-            int fileCount = reader.ReadInt32();
-
             // Read the manifest entries
             for (long i = 0; i < fileCount; i++)
             {

src/installer/test/HostActivationTests/BundledAppWithSubDirs.cs

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using Xunit;
+using Microsoft.DotNet.Cli.Build.Framework;
+
+namespace Microsoft.DotNet.CoreSetup.Test.HostActivation
+{
+    public class BundledAppWithSubDirs : IClassFixture<BundledAppWithSubDirs.SharedTestState>
+    {
+        private SharedTestState sharedTestState;
+
+        public BundledAppWithSubDirs(SharedTestState fixture)
+        {
+            sharedTestState = fixture;
+        }
+
+        [Fact]
+        private void Bundle_And_Run_App_With_Subdirs_Succeeds()
+        {
+            var fixture = sharedTestState.TestFixture.Copy();
+            var hostName = Path.GetFileName(fixture.TestProject.AppExe);
+
+            // Bundle to a single-file
+            // This step should be removed in favor of publishing with /p:PublishSingleFile=true
+            // once associated changes in SDK repo are checked in.
+            string singleFileDir = Path.Combine(fixture.TestProject.ProjectDirectory, "oneExe");
+            Directory.CreateDirectory(singleFileDir);
+            var bundler = new Microsoft.NET.HostModel.Bundle.Bundler(hostName, singleFileDir);
+            string singleFile = bundler.GenerateBundle(fixture.TestProject.OutputDirectory);
+
+            // Run the bundled app (extract files)
+            Command.Create(singleFile)
+                .CaptureStdErr()
+                .CaptureStdOut()
+                .Execute()
+                .Should()
+                .Pass()
+                .And
+                .HaveStdOutContaining("Wow! We now say hello to the big world and you.");
+
+            // Run the bundled app again (reuse extracted files)
+            Command.Create(singleFile)
+                .CaptureStdErr()
+                .CaptureStdOut()
+                .Execute()
+                .Should()
+                .Pass()
+                .And
+                .HaveStdOutContaining("Wow! We now say hello to the big world and you.");
+        }
+
+        public class SharedTestState : IDisposable
+        {
+            public TestProjectFixture TestFixture { get; set; }
+            public RepoDirectoriesProvider RepoDirectories { get; set; }
+
+            public SharedTestState()
+            {
+                RepoDirectories = new RepoDirectoriesProvider();
+
+                TestFixture = new TestProjectFixture("StandaloneAppWithSubDirs", RepoDirectories);
+                TestFixture
+                    .EnsureRestoredForRid(TestFixture.CurrentRid, RepoDirectories.CorehostPackages)
+                    .PublishProject(runtime: TestFixture.CurrentRid);
+            }
+
+            public void Dispose()
+            {
+                TestFixture.Dispose();
+            }
+        }
+    }
+}

src/installer/test/HostActivationTests/HostActivationTests.csproj

@@ -14,6 +14,8 @@
   
   <ItemGroup>
     <ProjectReference Include="..\TestUtils\TestUtils.csproj" />
+    <!-- This project reference should be removed once SDK changes to bundle via publish are available -->
+    <ProjectReference Include="..\..\managed\Microsoft.NET.HostModel\Microsoft.NET.HostModel.csproj" />
   </ItemGroup>
 
   <ItemGroup>