generate package config during runhooks (#19428)

The dart SDK needs to ship several packages which have opted into null safety for the tech preview. This requires the use of a package_config.json for compilation as this file contains the language versions. Use a dart script to generate the package_config file from the .packages file, and include that in the DEPS hooks section.

Fixes flutter/flutter#60628

DEPS

@@ -629,4 +629,12 @@ hooks = [
       'src/build/linux/sysroot_scripts/install-sysroot.py',
       '--arch=x64'],
   },
+  {
+    'name': 'dart package config',
+    'pattern': '.',
+    'action': [
+      'python',
+      'src/flutter/tools/run_third_party_dart.py',
+    ]
+  }
 ]

flutter_frontend_server/BUILD.gn

@@ -67,10 +67,9 @@ frontend_server_files +=
 application_snapshot("frontend_server") {
   main_dart = "bin/starter.dart"
   deps = [
-    ":package_incremental_compiler",
     "//flutter/lib/snapshot:kernel_platform_files",
   ]
-  dot_packages = rebase_path(".packages")
+  dot_packages = rebase_path(".dart_tool/package_config.json")
   flutter_patched_sdk = rebase_path("$root_out_dir/flutter_patched_sdk")
   training_args = [
     "--train",
@@ -80,27 +79,3 @@ application_snapshot("frontend_server") {
 
   inputs = frontend_server_files
 }
-
-# For flutter/flutter#36738 we make the source files available so that
-# we can generate a local frontend_server snapshot in the tools cache.
-action("package_incremental_compiler") {
-  script = "//flutter/flutter_frontend_server/package_incremental.py"
-
-  inputs = frontend_server_files
-
-  outputs = [
-    "$root_gen_dir/dart-pkg/flutter_frontend_server/pubspec.yaml",
-    "$root_gen_dir/dart-pkg/vm/pubspec.yaml",
-    "$root_gen_dir/dart-pkg/build_integration/pubspec.yaml",
-    "$root_gen_dir/dart-pkg/front_end/pubspec.yaml",
-    "$root_gen_dir/dart-pkg/kernel/pubspec.yaml",
-    "$root_gen_dir/dart-pkg/dev_compiler/pubspec.yaml",
-    "$root_gen_dir/dart-pkg/frontend_server/pubspec.yaml",
-  ]
-
-  args = [
-    "--input-root=" + rebase_path("//third_party/dart/pkg"),
-    "--output-root=" + rebase_path("$root_gen_dir/dart-pkg"),
-    "--frontend-server=" + rebase_path("//flutter"),
-  ]
-}

flutter_frontend_server/bin/starter.dart

+// @dart=2.8
 library frontend_server;
 
 import 'dart:io';

flutter_frontend_server/lib/server.dart

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// @dart=2.8
 library flutter_frontend_server;
 
 import 'dart:async';

flutter_frontend_server/package_incremental.py

-#!/usr/bin/env python
-# Copyright 2013 The Flutter Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import argparse
-import os
-import shutil
-import sys
-
-# The list of packages copied from the Dart SDK.
-PACKAGES = [
-  "vm",
-  "build_integration",
-  "kernel",
-  "front_end",
-  "dev_compiler",
-  "flutter_frontend_server",
-  "frontend_server",
-  "dev_compiler",
-]
-
-VM_PUBSPEC = r'''name: vm
-version: 0.0.1
-environment:
-  sdk: ">=2.2.2 <3.0.0"
-
-dependencies:
-  dev_compiler: any
-  front_end: any
-  kernel: any
-  meta: any
-  build_integration: any
-'''
-
-BUILD_INTEGRATION_PUBSPEC = r'''name: build_integration
-version: 0.0.1
-environment:
-  sdk: ">=2.2.2 <3.0.0"
-
-dependencies:
-  front_end: any
-  meta: any
-'''
-
-FLUTTER_FRONTEND_SERVER_PUBSPEC = r'''name: flutter_frontend_server
-version: 0.0.1
-environment:
-  sdk: ">=2.2.2 <3.0.0"
-
-dependencies:
-  args: any
-  path: any
-  vm: any
-'''
-
-KERNEL_PUBSPEC = r'''name: kernel
-version: 0.0.1
-environment:
-  sdk: '>=2.2.2 <3.0.0'
-
-dependencies:
-  args: any
-  meta: any
-'''
-
-FRONT_END_PUBSPEC = r'''name: front_end
-version: 0.0.1
-environment:
-  sdk: '>=2.2.2 <3.0.0'
-dependencies:
-  kernel: any
-  package_config: any
-  meta: any
-'''
-
-DEV_COMPILER_PUBSPEC = r'''name: dev_compiler
-version: 0.0.1
-environment:
-  sdk: '>=2.2.2 <3.0.0'
-dependencies:
-  analyzer: any
-  bazel_worker: any
-  build_integration: any
-  cli_util: any
-  source_maps: any
-'''
-
-FRONTEND_SERVER_PUBSPEC = r'''name: frontend_server
-version: 0.0.1
-environment:
-  sdk: '>=2.2.2 < 3.0.0'
-dependencies:
-  build_integration: any
-  vm: any
-  dev_compiler: any
-  front_end: any
-  kernel: any
-  args: any
-'''
-
-PUBSPECS = {
-  'vm': VM_PUBSPEC,
-  'build_integration': BUILD_INTEGRATION_PUBSPEC,
-  'flutter_frontend_server': FLUTTER_FRONTEND_SERVER_PUBSPEC,
-  'kernel': KERNEL_PUBSPEC,
-  'front_end': FRONT_END_PUBSPEC,
-  'dev_compiler': DEV_COMPILER_PUBSPEC,
-  'frontend_server': FRONTEND_SERVER_PUBSPEC,
-}
-
-def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--frontend-server', type=str, dest='frontend', action='store')
-  parser.add_argument('--input-root', type=str, dest='input', action='store')
-  parser.add_argument('--output-root', type=str, dest='output', action='store')
-
-  args = parser.parse_args()
-  for package in PACKAGES:
-    base = args.input
-    # Handle different path for frontend_server
-    if package == 'flutter_frontend_server':
-      base = args.frontend
-    package_root = os.path.join(base, package)
-    for root, directories, files in os.walk(package_root):
-      # We only care about actual source files, not generated code or tests.
-      for skip_dir in ['.git', 'gen', 'test']:
-        if skip_dir in directories:
-          directories.remove(skip_dir)
-
-      # Ensure we have a dest directory
-      if not os.path.isdir(os.path.join(args.output, package)):
-        os.makedirs(os.path.join(args.output, package))
-
-      for filename in files:
-        if filename.endswith('.dart') and not filename.endswith('_test.dart'):
-          destination_file = os.path.join(args.output, package,
-                                          os.path.relpath(os.path.join(root, filename), start=package_root))
-          parent_path = os.path.abspath(os.path.join(destination_file, os.pardir))
-          if not os.path.isdir(parent_path):
-            os.makedirs(parent_path)
-          shutil.copyfile(os.path.join(root, filename), destination_file)
-
-    # Write the overriden pubspec for each package.
-    pubspec_file = os.path.join(args.output, package, 'pubspec.yaml')
-    with open(pubspec_file, 'w+') as output_file:
-      output_file.write(PUBSPECS[package])
-
-if __name__ == '__main__':
-  sys.exit(main())

tools/generate_package_config/bin/generate_from_legacy.dart

+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+// @dart=2.8
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:package_config/package_config.dart';
+import 'package:package_config/packages_file.dart'; // ignore: deprecated_member_use
+import 'package:package_config/src/package_config_json.dart';
+import 'package:pub_semver/pub_semver.dart';
+import 'package:yaml/yaml.dart';
+
+void main(List<String> args) async {
+  if (args.length != 1) {
+    throw ArgumentError('Unexpected arguments $args\n\n$usage');
+  }
+  if (args.first == 'help') {
+    print(usage);
+    return;
+  }
+  var packagesUri = Uri.base.resolve(args.first);
+  var packagesFile = File.fromUri(packagesUri);
+  if (!await packagesFile.exists()) {
+    throw ArgumentError('Unable to read file at `$packagesUri`');
+  }
+  var packageMap = parse(await packagesFile.readAsBytes(), packagesFile.uri);
+  var packages = <Package>[];
+  for (var packageEntry in packageMap.entries) {
+    var name = packageEntry.key;
+    var uri = packageEntry.value;
+    if (uri.scheme != 'file') {
+      throw ArgumentError(
+          'Only file: schemes are supported, but the `$name` package has '
+          'the following uri: ${uri}');
+    }
+    Uri packageRoot;
+    Uri pubspec;
+    LanguageVersion languageVersion;
+    // Pub packages will always have a path ending in `lib/`, and a pubspec
+    // directly above that.
+    if (uri.path.endsWith('lib/')) {
+      packageRoot = uri.resolve('../');
+      pubspec = packageRoot.resolve('pubspec.yaml');
+      if (!File.fromUri(pubspec).existsSync()) {
+        continue;
+      }
+      // Default to 2.8 if not found to prevent all packages from accidentally
+      // opting into NNBD.
+      languageVersion = await languageVersionFromPubspec(pubspec, name) ??
+          LanguageVersion(2, 8);
+      packages.add(Package(name, packageRoot,
+          languageVersion: languageVersion, packageUriRoot: uri));
+    }
+  }
+  var outputFile =
+      File.fromUri(packagesFile.uri.resolve('.dart_tool/package_config.json'));
+  if (!outputFile.parent.existsSync()) {
+    outputFile.parent.createSync();
+  }
+  var baseUri = outputFile.uri;
+  var sink = outputFile.openWrite(encoding: utf8);
+  writePackageConfigJsonUtf8(PackageConfig(packages), baseUri, sink);
+  await sink.close();
+}
+
+const usage = 'Usage: pub run package_config:generate_from_legacy <input-file>';
+
+Future<LanguageVersion> languageVersionFromPubspec(
+    Uri pubspec, String packageName) async {
+  var pubspecFile = File.fromUri(pubspec);
+  if (!await pubspecFile.exists()) {
+    return null;
+  }
+  var pubspecYaml =
+      loadYaml(await pubspecFile.readAsString(), sourceUrl: pubspec) as YamlMap;
+
+  // Find the sdk constraint, or return null if none is present
+  var environment = pubspecYaml['environment'] as YamlMap;
+  if (environment == null) {
+    return null;
+  }
+  var sdkConstraint = environment['sdk'] as String;
+  if (sdkConstraint == null) {
+    return null;
+  }
+
+  var parsedConstraint = VersionConstraint.parse(sdkConstraint);
+  var min = parsedConstraint is Version
+      ? parsedConstraint
+      : parsedConstraint is VersionRange
+          ? parsedConstraint.min
+          : throw 'Unsupported version constraint type $parsedConstraint';
+
+  return LanguageVersion(min.major, min.minor);
+}

tools/generate_package_config/pubspec.yaml

+name: generate_package_config
+environment:
+  sdk: ">=2.8.0 <3.0.0"
+
+dependencies:
+  yaml: any
+  pub_semver: any
+  package_config: any

tools/run_third_party_dart.py

+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import subprocess
+
+if __name__ == '__main__':
+    subprocess.call(["src/third_party/dart/tools/sdks/dart-sdk/bin/dart", "pub", "global", "activate", "-spath", "./src/flutter/tools/generate_package_config"])
+    subprocess.call(["src/third_party/dart/tools/sdks/dart-sdk/bin/dart", "pub", "global", "run", "generate_package_config:generate_from_legacy", "src/flutter/flutter_frontend_server/.packages"])