Create a package-able incremental compiler (#12681)

frontend_server/BUILD.gn

@@ -40,9 +40,26 @@ if (is_fuchsia_host || is_fuchsia) {
 } else {
   import("//third_party/dart/utils/application_snapshot.gni")
 
+  frontend_server_files =
+      exec_script("//third_party/dart/tools/list_dart_files.py",
+                  [
+                    "absolute",
+                    rebase_path("."),
+                  ],
+                  "list lines")
+
+  frontend_server_files +=
+      exec_script("//third_party/dart/tools/list_dart_files.py",
+                  [
+                    "absolute",
+                    rebase_path("../../third_party/dart/pkg"),
+                  ],
+                  "list lines")
+
   application_snapshot("frontend_server") {
     main_dart = "bin/starter.dart"
     deps = [
+      ":package_incremental_compiler",
       "$flutter_root/lib/snapshot:kernel_platform_files",
     ]
     dot_packages = rebase_path(".packages")
@@ -53,22 +70,28 @@ if (is_fuchsia_host || is_fuchsia) {
       rebase_path(main_dart),
     ]
 
-    frontend_server_files =
-        exec_script("//third_party/dart/tools/list_dart_files.py",
-                    [
-                      "absolute",
-                      rebase_path("."),
-                    ],
-                    "list lines")
-
-    frontend_server_files +=
-        exec_script("//third_party/dart/tools/list_dart_files.py",
-                    [
-                      "absolute",
-                      rebase_path("../../third_party/dart/pkg"),
-                    ],
-                    "list lines")
+    inputs = frontend_server_files
+  }
+
+  # For flutter/flutter#36738 we make the source files available so that
+  # we can generate a local frontend_server snapshot in the tools cache.
+  action("package_incremental_compiler") {
+    script = "$flutter_root/frontend_server/package_incremental.py"
 
     inputs = frontend_server_files
+
+    outputs = [
+      "$root_build_dir/dist/packages/frontend_server/pubspec.yaml",
+      "$root_build_dir/dist/packages/vm/pubspec.yaml",
+      "$root_build_dir/dist/packages/build_integration/pubspec.yaml",
+
+      "$root_build_dir/dist/packages/front_end/pubspec.yaml",
+      "$root_build_dir/dist/packages/kernel/pubspec.yaml",
+    ]
+
+    args = [
+      "--input-root=" + rebase_path("//third_party/dart/pkg"),
+      "--output-root=" + rebase_path("$root_gen_dir/dart-pkg"),
+    ]
   }
 }

frontend_server/lib/server.dart

@@ -11,21 +11,30 @@ import 'package:args/args.dart';
 import 'package:path/path.dart' as path;
 
 import 'package:vm/incremental_compiler.dart';
-import 'package:vm/frontend_server.dart' as frontend show FrontendCompiler,
-    CompilerInterface, listenAndCompile, argParser, usage;
+import 'package:vm/frontend_server.dart' as frontend
+    show
+        FrontendCompiler,
+        CompilerInterface,
+        listenAndCompile,
+        argParser,
+        usage,
+        ProgramTransformer;
 
 /// Wrapper around [FrontendCompiler] that adds [widgetCreatorTracker] kernel
 /// transformation to the compilation.
-class _FlutterFrontendCompiler implements frontend.CompilerInterface{
+class _FlutterFrontendCompiler implements frontend.CompilerInterface {
   final frontend.CompilerInterface _compiler;
 
   _FlutterFrontendCompiler(StringSink output,
-      {bool unsafePackageSerialization}) :
-          _compiler = frontend.FrontendCompiler(output,
-          unsafePackageSerialization: unsafePackageSerialization);
+      {bool unsafePackageSerialization,
+      frontend.ProgramTransformer transformer})
+      : _compiler = frontend.FrontendCompiler(output,
+            transformer: transformer,
+            unsafePackageSerialization: unsafePackageSerialization);
 
   @override
-  Future<bool> compile(String filename, ArgResults options, {IncrementalCompiler generator}) async {
+  Future<bool> compile(String filename, ArgResults options,
+      {IncrementalCompiler generator}) async {
     return _compiler.compile(filename, options, generator: generator);
   }
 
@@ -57,8 +66,8 @@ class _FlutterFrontendCompiler implements frontend.CompilerInterface{
       String libraryUri,
       String klass,
       bool isStatic) {
-    return _compiler.compileExpression(expression, definitions, typeDefinitions,
-        libraryUri, klass, isStatic);
+    return _compiler.compileExpression(
+        expression, definitions, typeDefinitions, libraryUri, klass, isStatic);
   }
 
   @override
@@ -77,11 +86,12 @@ class _FlutterFrontendCompiler implements frontend.CompilerInterface{
 /// `compiler` is an optional parameter so it can be replaced with mocked
 /// version for testing.
 Future<int> starter(
-    List<String> args, {
-      frontend.CompilerInterface compiler,
-      Stream<List<int>> input,
-      StringSink output,
-    }) async {
+  List<String> args, {
+  frontend.CompilerInterface compiler,
+  Stream<List<int>> input,
+  StringSink output,
+  frontend.ProgramTransformer transformer,
+}) async {
   ArgResults options;
   try {
     options = frontend.argParser.parse(args);
@@ -98,7 +108,8 @@ Future<int> starter(
 
     final String input = options.rest[0];
     final String sdkRoot = options['sdk-root'];
-    final Directory temp = Directory.systemTemp.createTempSync('train_frontend_server');
+    final Directory temp =
+        Directory.systemTemp.createTempSync('train_frontend_server');
     try {
       final String outputTrainingDill = path.join(temp.path, 'app.dill');
       options = frontend.argParser.parse(<String>[
@@ -126,6 +137,7 @@ Future<int> starter(
   }
 
   compiler ??= _FlutterFrontendCompiler(output,
+      transformer: transformer,
       unsafePackageSerialization: options['unsafe-package-serialization']);
 
   if (options.rest.isNotEmpty) {

frontend_server/package_incremental.py

+#!/usr/bin/env python
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import os
+import shutil
+import sys
+
+# The list of packages copied from the Dart SDK.
+PACKAGES = [
+  "vm", "build_integration", "kernel", "front_end", "frontend_server",
+]
+
+VM_PUBSPEC = r'''name: vm
+version: 0.0.1
+environment:
+  sdk: ">=2.2.2 <3.0.0"
+
+dependencies:
+  front_end: any
+  kernel: any
+  meta: any
+  build_integration: any
+'''
+
+BUILD_INTEGRATION_PUBSPEC = r'''name: build_integration
+version: 0.0.1
+environment:
+  sdk: ">=2.2.2 <3.0.0"
+
+dependencies:
+  front_end: any
+  meta: any
+'''
+
+FRONTEND_SERVER_PUBSPEC = r'''name: frontend_server
+version: 0.0.1
+environment:
+  sdk: ">=2.2.2 <3.0.0"
+
+dependencies:
+  args: any
+  path: any
+  vm: any
+'''
+
+KERNEL_PUBSPEC = r'''name: kernel
+version: 0.0.1
+environment:
+  sdk: '>=2.2.2 <3.0.0'
+
+dependencies:
+  args: any
+  meta: any
+'''
+
+FRONT_END_PUBSPEC = r'''name: front_end
+version: 0.0.1
+environment:
+  sdk: '>=2.2.2 <3.0.0'
+dependencies:
+  kernel: any
+  package_config: any
+  meta: any
+'''
+
+PUBSPECS = {
+  'vm': VM_PUBSPEC,
+  'build_integration': BUILD_INTEGRATION_PUBSPEC,
+  'frontend_server': FRONTEND_SERVER_PUBSPEC,
+  'kernel': KERNEL_PUBSPEC,
+  'front_end': FRONT_END_PUBSPEC,
+}
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--input-root', type=str, dest='input', action='store')
+  parser.add_argument('--output-root', type=str, dest='output', action='store')
+
+  args = parser.parse_args()
+  for package in PACKAGES:
+    package_root = os.path.join(args.input, package)
+    for root, directories, files in os.walk(package_root):
+      # We only care about actual source files, not generated code or tests.
+      for skip_dir in ['.git', 'gen', 'test']:
+        if skip_dir in directories:
+          directories.remove(skip_dir)
+
+      # Ensure we have a dest directory
+      if not os.path.isdir(os.path.join(args.output, package)):
+        os.makedirs(os.path.join(args.output, package))
+
+      for filename in files:
+        if filename.endswith('.dart') and not filename.endswith('_test.dart'):
+          destination_file = os.path.join(args.output, package,
+                                          os.path.relpath(os.path.join(root, filename), start=package_root))
+          parent_path = os.path.abspath(os.path.join(destination_file, os.pardir))
+          if not os.path.isdir(parent_path):
+            os.makedirs(parent_path)
+          shutil.copyfile(os.path.join(root, filename), destination_file)
+
+          # Write the overriden pubspec for each package.
+          pubspec_file = os.path.join(args.output, package, 'pubspec.yaml')
+          with open(pubspec_file, 'w+') as output_file:
+            output_file.write(PUBSPECS[package])
+
+if __name__ == '__main__':
+  sys.exit(main())

frontend_server/pubspec.yaml

@@ -4,6 +4,10 @@ description: Communication pipe to Dart Frontend
 homepage: http://flutter.io
 author: Flutter Authors <flutter-dev@googlegroups.com>
 
+environment:
+  # The pub client defaults to an <2.0.0 sdk constraint which we need to explicitly overwrite.
+  sdk: ">=2.2.2 <3.0.0"
+
 dependencies:
   args: any
   async: any
@@ -22,36 +26,3 @@ dependencies:
   typed_data: any
   usage: any
   vm: any
-
-dev_dependencies:
-  analyzer: any
-  boolean_selector: any
-  cli_util: any
-  csslib: any
-  glob: any
-  html: any
-  http: any
-  http_multi_server: any
-  http_parser: any
-  matcher: any
-  mime: any
-  mockito: any
-  package_resolver: any
-  plugin: any
-  pool: any
-  pub_semver: any
-  shelf: any
-  shelf_packages_handler: any
-  shelf_static: any
-  shelf_web_socket: any
-  source_map_stack_trace: any
-  source_maps: any
-  stack_trace: any
-  stream_channel: any
-  string_scanner: any
-  test: any
-  utf: any
-  watcher: any
-  web_socket_channel: any
-  when: any
-  yaml: any

frontend_server/test/server_test.dart

-import 'dart:async';
-
-import 'package:frontend_server/server.dart';
-import 'package:mockito/mockito.dart';
-import 'package:test/test.dart';
-import 'package:vm/frontend_server.dart' as frontend show CompilerInterface;
-
-class _MockedCompiler extends Mock implements frontend.CompilerInterface {}
-
-Future<int> main() async {
-  group('basic', () {
-    final frontend.CompilerInterface compiler = _MockedCompiler();
-
-    test('train with mocked compiler completes', () async {
-      expect(await starter(<String>['--train'], compiler: compiler), equals(0));
-    });
-  });
-  return 0;
-}