Rewrite tools/format.py in deno (#1528)

Note: findFiles and findFilesWalk are borrowed from the previous
attempt of @pseudo-su (#1434)

.github/CONTRIBUTING.md

@@ -18,7 +18,7 @@ Docs are [here](https://github.com/denoland/deno/blob/master/Docs.md).
 Before submitting, please make sure the following is done:
 
 1. Ensure `./tools/test.py` passes.
-2. Format your code with `./tools/format.py`.
+2. Format your code with `./tools/format.ts`.
 3. Make sure `./tools/lint.py` passes.
 
 ## Changes to `third_party`

.travis.yml

@@ -64,8 +64,8 @@ before_script:
 # Default script for release builds.
 script:
 - ./tools/lint.py
-- ./tools/test_format.py
 - ./tools/build.py -C target/release
+- ./tools/test_format.py
 - DENO_BUILD_MODE=release ./tools/test.py
 
 jobs:

Docs.md

@@ -344,7 +344,7 @@ submodule. However, you need to install separately:
     ./tools/test.py
 
     # Format code.
-    ./tools/format.py
+    deno ./tools/format.ts
 
 Other useful commands:
 

js/unit_tests.ts

@@ -42,4 +42,6 @@ import "./url_test.ts";
 import "./url_search_params_test.ts";
 import "./write_file_test.ts";
 
+import "../tools/util_test.ts";
+
 import "../website/app_test.js";

tools/format.py

-#!/usr/bin/env python
-# Copyright 2018 the Deno authors. All rights reserved. MIT license.
-from glob import glob
-import os
-import sys
-from third_party import fix_symlinks, google_env, python_env
-from third_party import clang_format_path, third_party_path
-from util import root_path, run, find_exts, platform
-
-fix_symlinks()
-
-prettier = os.path.join(third_party_path, "node_modules", "prettier",
-                        "bin-prettier.js")
-tools_path = os.path.join(root_path, "tools")
-rustfmt_config = os.path.join(tools_path, "rustfmt.toml")
-
-os.chdir(root_path)
-
-
-def qrun(cmd, env=None):
-    run(cmd, quiet=True, env=env)
-
-
-print "clang_format"
-qrun([clang_format_path, "-i", "-style", "Google"] +
-     find_exts(["libdeno"], [".cc", ".h"]))
-
-print "gn format"
-for fn in ["BUILD.gn", ".gn"] + find_exts(["build_extra", "libdeno"],
-                                          [".gn", ".gni"]):
-    qrun(["third_party/depot_tools/gn", "format", fn], env=google_env())
-
-print "yapf"
-qrun(
-    [sys.executable, "third_party/python_packages/bin/yapf", "-i"] + find_exts(
-        ["tools", "build_extra"], [".py"], skip=["tools/clang"]),
-    env=python_env())
-
-print "prettier"
-qrun(["node", prettier, "--write", "--loglevel=error"] + ["rollup.config.js"] +
-     glob("*.json") + glob("*.md") +
-     find_exts([".github", "js", "tests", "tools", "website"],
-               [".js", ".json", ".ts", ".md"],
-               skip=["tools/clang", "js/deps"]))
-
-print "rustfmt"
-qrun([
-    "third_party/rustfmt/" + platform() +
-    "/rustfmt", "--config-path", rustfmt_config, "build.rs"
-] + find_exts(["src"], [".rs"]))

tools/format.ts

+#!/usr/bin/env deno --allow-run
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+import * as deno from "deno";
+import { join } from "../js/deps/https/deno.land/x/std/path/index.ts";
+import { findFiles } from "./util.ts";
+
+const clangFormat = join("third_party", "depot_tools", "clang-format");
+const gn = join("third_party", "depot_tools", "gn");
+const prettier = join(
+  "third_party",
+  "node_modules",
+  "prettier",
+  "bin-prettier.js"
+);
+const yapf = join("third_party", "python_packages", "bin", "yapf");
+const rustfmt = join("third_party", "rustfmt", deno.platform.os, "rustfmt");
+const rustfmtConfig = join("tools", "rustfmt.toml");
+
+const run = (...args: string[]) => {
+  if (deno.platform.os === "win") {
+    args = ["cmd.exe", "/c", ...args];
+  }
+  return deno.run({ args, stdout: "null", stderr: "piped" }).status();
+};
+
+(async () => {
+  console.log("clang_format");
+  await run(
+    clangFormat,
+    "-i",
+    "-style",
+    "Google",
+    ...findFiles(["libdeno"], [".cc", ".h"])
+  );
+
+  console.log("gn format");
+  for (const fn of [
+    "BUILD.gn",
+    ".gn",
+    ...findFiles(["build_extra", "libdeno"], [".gn", ".gni"])
+  ]) {
+    await run(gn, "format", fn);
+  }
+
+  console.log("yapf");
+  await run(
+    "python",
+    yapf,
+    "-i",
+    ...findFiles(["tools", "build_extra"], [".py"], {
+      skip: [join("tools", "clang")]
+    })
+  );
+
+  console.log("prettier");
+  await run(
+    "node",
+    prettier,
+    "--write",
+    "--loglevel=error",
+    "rollup.config.js",
+    ...findFiles(["."], [".json", ".md"], { depth: 1 }),
+    ...findFiles(
+      [".github", "js", "tests", "tools", "website"],
+      [".js", ".json", ".ts", ".md"],
+      { skip: [join("tools", "clang"), join("js", "deps")] }
+    )
+  );
+
+  console.log("rustfmt");
+  await run(
+    rustfmt,
+    "--config-path",
+    rustfmtConfig,
+    "build.rs",
+    ...findFiles(["src"], [".rs"])
+  );
+})();

tools/test_format.py

 #!/usr/bin/env python
-# This program fails if ./tools/format.py changes any files.
+# Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+# This program fails if ./tools/format.ts changes any files.
 
+import os
 import sys
 import util
 import sys
 import subprocess
+from distutils.spawn import find_executable
+
+
+def lookup_deno_path():
+    deno_exe = "deno" + util.executable_suffix
+    release_deno = os.path.join(util.root_path, "target", "release", deno_exe)
+    debug_deno = os.path.join(util.root_path, "target", "debug", deno_exe)
+
+    if os.path.exists(release_deno):
+        return release_deno
+    if os.path.exists(debug_deno):
+        return debug_deno
+
+    return find_executable("deno")
 
 
 def main():
-    util.run([sys.executable, "tools/format.py"])
+    deno_path = lookup_deno_path()
+
+    if not deno_path:
+        print "No available deno executable."
+        sys.exit(1)
+
+    util.run([deno_path, "--allow-run", "tools/format.ts"])
     output = util.run_output(
         ["git", "status", "-uno", "--porcelain", "--ignore-submodules"])
     if len(output) > 0:
-        print "Run tools/format.py "
+        print "Run tools/format.ts "
         print output
         sys.exit(1)
 

tools/testdata/find_files_testdata/bar.md

+# bar

tools/testdata/find_files_testdata/bar.ts

+console.log("bar");

tools/testdata/find_files_testdata/bar.txt

+bar

tools/testdata/find_files_testdata/foo.md

+# foo

tools/testdata/find_files_testdata/foo.ts

+console.log("foo");

tools/testdata/find_files_testdata/foo.txt

+foo

tools/testdata/find_files_testdata/subdir0/bar.ts

+console.log("bar");

tools/testdata/find_files_testdata/subdir0/foo.ts

+console.log("foo");

tools/testdata/find_files_testdata/subdir0/subdir0/bar.ts

+console.log("bar");

tools/testdata/find_files_testdata/subdir0/subdir0/foo.ts

+console.log("foo");

tools/testdata/find_files_testdata/subdir1/bar.ts

+console.log("bar");

tools/testdata/find_files_testdata/subdir1/foo.ts

+console.log("foo");

tools/util.ts

+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+import { lstatSync, readDirSync } from "deno";
+
+export interface FindOptions {
+  skip?: string[];
+  depth?: number;
+}
+
+/**
+ * Finds files of the give extensions under the given paths recursively.
+ * @param dirs directories
+ * @param exts extensions
+ * @param skip patterns to ignore
+ * @param depth depth to find
+ */
+export function findFiles(
+  dirs: string[],
+  exts: string[],
+  { skip = [], depth = 20 }: FindOptions = {}
+) {
+  return findFilesWalk(dirs, depth).filter(
+    path =>
+      exts.some(ext => path.endsWith(ext)) &&
+      skip.every(pattern => !path.includes(pattern))
+  );
+}
+
+function findFilesWalk(paths: string[], depth: number) {
+  if (depth < 0) {
+    return [];
+  }
+
+  const foundPaths = paths.map(path =>
+    lstatSync(path).isDirectory()
+      ? findFilesWalk(readDirSync(path).map(f => f.path), depth - 1)
+      : path
+  );
+
+  return [].concat(...foundPaths);
+}

tools/util_test.ts

+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+import { test, assertEqual } from "../js/test_util.ts";
+import { findFiles } from "./util.ts";
+
+const testDir = "tools/testdata/find_files_testdata";
+
+// Sorts and replace backslashes with slashes.
+const normalize = files => files.map(f => f.replace(/\\/g, "/")).sort();
+
+test(function testFindFiles() {
+  const files = findFiles([testDir], [".ts", ".md"]);
+  assertEqual(normalize(files), [
+    `${testDir}/bar.md`,
+    `${testDir}/bar.ts`,
+    `${testDir}/foo.md`,
+    `${testDir}/foo.ts`,
+    `${testDir}/subdir0/bar.ts`,
+    `${testDir}/subdir0/foo.ts`,
+    `${testDir}/subdir0/subdir0/bar.ts`,
+    `${testDir}/subdir0/subdir0/foo.ts`,
+    `${testDir}/subdir1/bar.ts`,
+    `${testDir}/subdir1/foo.ts`
+  ]);
+});
+
+test(function testFindFilesDepth() {
+  const files = findFiles([testDir], [".ts", ".md"], { depth: 1 });
+  assertEqual(normalize(files), [
+    `${testDir}/bar.md`,
+    `${testDir}/bar.ts`,
+    `${testDir}/foo.md`,
+    `${testDir}/foo.ts`
+  ]);
+});
+
+test(function testFindFilesSkip() {
+  const files = findFiles([testDir], [".ts", ".md"], {
+    skip: ["foo.md", "subdir1"]
+  });
+  assertEqual(normalize(files), [
+    `${testDir}/bar.md`,
+    `${testDir}/bar.ts`,
+    `${testDir}/foo.ts`,
+    `${testDir}/subdir0/bar.ts`,
+    `${testDir}/subdir0/foo.ts`,
+    `${testDir}/subdir0/subdir0/bar.ts`,
+    `${testDir}/subdir0/subdir0/foo.ts`
+  ]);
+});