提交 2f90225c 编写于 作者: N Nayeem Rahman 提交者: Ryan Dahl

Implement expandGlob() and expandGlobSync() (denoland/deno_std#617)

fs/glob.ts:
- Improve prototypes for expandGlob() and expandGlobSync() from denoland/deno_std#604.
- Rename glob() to globToRegExp().
- Add normalizeGlob() and joinGlobs().
- Extract GlobToRegExpOptions from GlobOptions, remove the strict
  and filepath options.

fs/globrex.ts:
- Add GlobrexOptions.

fs/path/constants.ts:
- Add SEP_PATTERN.

fs/walk.ts:
- Add WalkOptions::includeFiles
- Default WalkOptions::includeDirs to true.
- Don't traverse directories matching a skip pattern.
- Remove walkSync()'s default root value.

prettier:
- Refactor to use expandGlob().

testing:
- Make findTestModules() an async generator.
Original: https://github.com/denoland/deno_std/commit/8c90bd9d0b1c78b023d36462ffaa9446ef22490c
上级 aca22533
......@@ -14,9 +14,9 @@ async function main(opts): Promise<void> {
"--ignore",
"node_modules",
"--ignore",
"testdata",
"**/testdata",
"--ignore",
"vendor",
"**/vendor",
"--write"
];
......
......@@ -100,16 +100,16 @@ exists("./foo"); // returns a Promise<boolean>
existsSync("./foo"); // returns boolean
```
### glob
### globToRegExp
Generate a regex based on glob pattern and options
This was meant to be using the the `fs.walk` function
but can be used anywhere else.
```ts
import { glob } from "https://deno.land/std/fs/mod.ts";
import { globToRegExp } from "https://deno.land/std/fs/mod.ts";
glob("foo/**/*.json", {
globToRegExp("foo/**/*.json", {
flags: "g",
extended: true,
globstar: true
......@@ -160,7 +160,7 @@ Iterate all files in a directory recursively.
```ts
import { walk, walkSync } from "https://deno.land/std/fs/mod.ts";
for (const fileInfo of walkSync()) {
for (const fileInfo of walkSync(".")) {
console.log(fileInfo.filename);
}
......
import { globrex } from "./globrex.ts";
import { isAbsolute, join } from "./path/mod.ts";
import { SEP, SEP_PATTERN, isWindows } from "./path/constants.ts";
import { isAbsolute, join, normalize } from "./path/mod.ts";
import { WalkInfo, walk, walkSync } from "./walk.ts";
const { cwd } = Deno;
const { DenoError, ErrorKind, cwd, stat, statSync } = Deno;
type FileInfo = Deno.FileInfo;
export interface GlobOptions {
// Allow ExtGlob features
extended?: boolean;
// When globstar is true, '/foo/**' is equivelant
// to '/foo/*' when globstar is false.
// Having globstar set to true is the same usage as
// using wildcards in bash
globstar?: boolean;
// be laissez faire about mutiple slashes
strict?: boolean;
// Parse as filepath for extra path related features
filepath?: boolean;
// Flag to use in the generated RegExp
}
export interface GlobToRegExpOptions extends GlobOptions {
flags?: string;
}
......@@ -27,12 +22,12 @@ export interface GlobOptions {
*
* Looking for all the `ts` files:
* walkSync(".", {
* match: [glob("*.ts")]
* match: [globToRegExp("*.ts")]
* })
*
* Looking for all the `.json` files in any subfolder:
* walkSync(".", {
* match: [glob(join("a", "**", "*.json"),{
* match: [globToRegExp(join("a", "**", "*.json"),{
* flags: "g",
* extended: true,
* globstar: true
......@@ -43,12 +38,12 @@ export interface GlobOptions {
* @param options - Specific options for the glob pattern
* @returns A RegExp for the glob pattern
*/
export function glob(glob: string, options: GlobOptions = {}): RegExp {
const result = globrex(glob, options);
if (options.filepath) {
return result.path!.regex;
}
return result.regex;
export function globToRegExp(
glob: string,
options: GlobToRegExpOptions = {}
): RegExp {
const result = globrex(glob, { ...options, strict: false, filepath: true });
return result.path!.regex;
}
/** Test whether the given string is a glob */
......@@ -84,11 +79,78 @@ export function isGlob(str: string): boolean {
return false;
}
/** Like normalize(), but doesn't collapse "**\/.." when `globstar` is true. */
export function normalizeGlob(
glob: string,
{ globstar = false }: GlobOptions = {}
): string {
if (!!glob.match(/\0/g)) {
throw new DenoError(
ErrorKind.InvalidPath,
`Glob contains invalid characters: "${glob}"`
);
}
if (!globstar) {
return normalize(glob);
}
const s = SEP_PATTERN.source;
const badParentPattern = new RegExp(
`(?<=(${s}|^)\\*\\*${s})\\.\\.(?=${s}|$)`,
"g"
);
return normalize(glob.replace(badParentPattern, "\0")).replace(/\0/g, "..");
}
/** Like join(), but doesn't collapse "**\/.." when `globstar` is true. */
export function joinGlobs(
globs: string[],
{ extended = false, globstar = false }: GlobOptions = {}
): string {
if (!globstar || globs.length == 0) {
return join(...globs);
}
if (globs.length === 0) return ".";
let joined: string | undefined;
for (const glob of globs) {
let path = glob;
if (path.length > 0) {
if (!joined) joined = path;
else joined += `${SEP}${path}`;
}
}
if (!joined) return ".";
return normalizeGlob(joined, { extended, globstar });
}
export interface ExpandGlobOptions extends GlobOptions {
root?: string;
exclude?: string[];
includeDirs?: boolean;
}
interface SplitPath {
segments: string[];
isAbsolute: boolean;
hasTrailingSep: boolean;
// Defined for any absolute Windows path.
winRoot?: string;
}
// TODO: Maybe make this public somewhere.
function split(path: string): SplitPath {
const s = SEP_PATTERN.source;
const segments = path
.replace(new RegExp(`^${s}|${s}$`, "g"), "")
.split(SEP_PATTERN);
const isAbsolute_ = isAbsolute(path);
return {
segments,
isAbsolute: isAbsolute_,
hasTrailingSep: !!path.match(new RegExp(`${s}$`)),
winRoot: isWindows && isAbsolute_ ? segments.shift() : undefined
};
}
/**
* Expand the glob string from the specified `root` directory and yield each
* result as a `WalkInfo` object.
......@@ -97,47 +159,203 @@ export interface ExpandGlobOptions extends GlobOptions {
// This is a very incomplete solution. The whole directory tree from `root` is
// walked and parent paths are not supported.
export async function* expandGlob(
globString: string,
glob: string,
{
root = cwd(),
exclude = [],
includeDirs = true,
extended = false,
globstar = false,
strict = false,
filepath = true,
flags = ""
globstar = false
}: ExpandGlobOptions = {}
): AsyncIterableIterator<WalkInfo> {
const absoluteGlob = isAbsolute(globString)
? globString
: join(root, globString);
const globOptions = { extended, globstar, strict, filepath, flags };
yield* walk(root, {
match: [glob(absoluteGlob, globOptions)],
includeDirs
});
const globOptions: GlobOptions = { extended, globstar };
const absRoot = isAbsolute(root)
? normalize(root)
: joinGlobs([cwd(), root], globOptions);
const resolveFromRoot = (path: string): string =>
isAbsolute(path)
? normalize(path)
: joinGlobs([absRoot, path], globOptions);
const excludePatterns = exclude
.map(resolveFromRoot)
.map((s: string): RegExp => globToRegExp(s, globOptions));
const shouldInclude = ({ filename }: WalkInfo): boolean =>
!excludePatterns.some((p: RegExp): boolean => !!filename.match(p));
const { segments, hasTrailingSep, winRoot } = split(resolveFromRoot(glob));
let fixedRoot = winRoot != undefined ? winRoot : "/";
while (segments.length > 0 && !isGlob(segments[0])) {
fixedRoot = joinGlobs([fixedRoot, segments.shift()!], globOptions);
}
let fixedRootInfo: WalkInfo;
try {
fixedRootInfo = { filename: fixedRoot, info: await stat(fixedRoot) };
} catch {
return;
}
async function* advanceMatch(
walkInfo: WalkInfo,
globSegment: string
): AsyncIterableIterator<WalkInfo> {
if (!walkInfo.info.isDirectory()) {
return;
} else if (globSegment == "..") {
const parentPath = joinGlobs([walkInfo.filename, ".."], globOptions);
try {
return yield* [
{ filename: parentPath, info: await stat(parentPath) }
].filter(shouldInclude);
} catch {
return;
}
} else if (globSegment == "**") {
return yield* walk(walkInfo.filename, {
includeFiles: false,
skip: excludePatterns
});
}
yield* walk(walkInfo.filename, {
maxDepth: 1,
match: [
globToRegExp(
joinGlobs([walkInfo.filename, globSegment], globOptions),
globOptions
)
],
skip: excludePatterns
});
}
let currentMatches: WalkInfo[] = [fixedRootInfo];
for (const segment of segments) {
// Advancing the list of current matches may introduce duplicates, so we
// pass everything through this Map.
const nextMatchMap: Map<string, FileInfo> = new Map();
for (const currentMatch of currentMatches) {
for await (const nextMatch of advanceMatch(currentMatch, segment)) {
nextMatchMap.set(nextMatch.filename, nextMatch.info);
}
}
currentMatches = [...nextMatchMap].sort().map(
([filename, info]): WalkInfo => ({
filename,
info
})
);
}
if (hasTrailingSep) {
currentMatches = currentMatches.filter(({ info }): boolean =>
info.isDirectory()
);
}
if (!includeDirs) {
currentMatches = currentMatches.filter(
({ info }): boolean => !info.isDirectory()
);
}
yield* currentMatches;
}
/** Synchronous version of `expandGlob()`. */
// TODO: As `expandGlob()`.
export function* expandGlobSync(
globString: string,
glob: string,
{
root = cwd(),
exclude = [],
includeDirs = true,
extended = false,
globstar = false,
strict = false,
filepath = true,
flags = ""
globstar = false
}: ExpandGlobOptions = {}
): IterableIterator<WalkInfo> {
const absoluteGlob = isAbsolute(globString)
? globString
: join(root, globString);
const globOptions = { extended, globstar, strict, filepath, flags };
yield* walkSync(root, {
match: [glob(absoluteGlob, globOptions)],
includeDirs
});
const globOptions: GlobOptions = { extended, globstar };
const absRoot = isAbsolute(root)
? normalize(root)
: joinGlobs([cwd(), root], globOptions);
const resolveFromRoot = (path: string): string =>
isAbsolute(path)
? normalize(path)
: joinGlobs([absRoot, path], globOptions);
const excludePatterns = exclude
.map(resolveFromRoot)
.map((s: string): RegExp => globToRegExp(s, globOptions));
const shouldInclude = ({ filename }: WalkInfo): boolean =>
!excludePatterns.some((p: RegExp): boolean => !!filename.match(p));
const { segments, hasTrailingSep, winRoot } = split(resolveFromRoot(glob));
let fixedRoot = winRoot != undefined ? winRoot : "/";
while (segments.length > 0 && !isGlob(segments[0])) {
fixedRoot = joinGlobs([fixedRoot, segments.shift()!], globOptions);
}
let fixedRootInfo: WalkInfo;
try {
fixedRootInfo = { filename: fixedRoot, info: statSync(fixedRoot) };
} catch {
return;
}
function* advanceMatch(
walkInfo: WalkInfo,
globSegment: string
): IterableIterator<WalkInfo> {
if (!walkInfo.info.isDirectory()) {
return;
} else if (globSegment == "..") {
const parentPath = joinGlobs([walkInfo.filename, ".."], globOptions);
try {
return yield* [
{ filename: parentPath, info: statSync(parentPath) }
].filter(shouldInclude);
} catch {
return;
}
} else if (globSegment == "**") {
return yield* walkSync(walkInfo.filename, {
includeFiles: false,
skip: excludePatterns
});
}
yield* walkSync(walkInfo.filename, {
maxDepth: 1,
match: [
globToRegExp(
joinGlobs([walkInfo.filename, globSegment], globOptions),
globOptions
)
],
skip: excludePatterns
});
}
let currentMatches: WalkInfo[] = [fixedRootInfo];
for (const segment of segments) {
// Advancing the list of current matches may introduce duplicates, so we
// pass everything through this Map.
const nextMatchMap: Map<string, FileInfo> = new Map();
for (const currentMatch of currentMatches) {
for (const nextMatch of advanceMatch(currentMatch, segment)) {
nextMatchMap.set(nextMatch.filename, nextMatch.info);
}
}
currentMatches = [...nextMatchMap].sort().map(
([filename, info]): WalkInfo => ({
filename,
info
})
);
}
if (hasTrailingSep) {
currentMatches = currentMatches.filter(({ info }): boolean =>
info.isDirectory()
);
}
if (!includeDirs) {
currentMatches = currentMatches.filter(
({ info }): boolean => !info.isDirectory()
);
}
yield* currentMatches;
}
const { cwd, mkdir } = Deno;
import { test, runIfMain } from "../testing/mod.ts";
import { assert, assertEquals } from "../testing/asserts.ts";
import { isWindows } from "./path/constants.ts";
import { SEP, isWindows } from "./path/constants.ts";
import {
ExpandGlobOptions,
expandGlob,
glob,
expandGlobSync,
globToRegExp,
isGlob,
expandGlobSync
joinGlobs,
normalizeGlob
} from "./glob.ts";
import { join, normalize, relative } from "./path.ts";
import { WalkInfo } from "./walk.ts";
import { testWalk } from "./walk_test.ts";
import { touch, walkArray } from "./walk_test.ts";
test({
name: "glob: glob to regex",
fn(): void {
assertEquals(glob("unicorn.*") instanceof RegExp, true);
assertEquals(glob("unicorn.*").test("poney.ts"), false);
assertEquals(glob("unicorn.*").test("unicorn.py"), true);
assertEquals(glob("*.ts").test("poney.ts"), true);
assertEquals(glob("*.ts").test("unicorn.js"), false);
assertEquals(globToRegExp("unicorn.*") instanceof RegExp, true);
assertEquals(globToRegExp("unicorn.*").test("poney.ts"), false);
assertEquals(globToRegExp("unicorn.*").test("unicorn.py"), true);
assertEquals(globToRegExp("*.ts").test("poney.ts"), true);
assertEquals(globToRegExp("*.ts").test("unicorn.js"), false);
assertEquals(
glob(join("unicorn", "**", "cathedral.ts")).test(
globToRegExp(join("unicorn", "**", "cathedral.ts")).test(
join("unicorn", "in", "the", "cathedral.ts")
),
true
);
assertEquals(
glob(join("unicorn", "**", "cathedral.ts")).test(
globToRegExp(join("unicorn", "**", "cathedral.ts")).test(
join("unicorn", "in", "the", "kitchen.ts")
),
false
);
assertEquals(
glob(join("unicorn", "**", "bathroom.*")).test(
globToRegExp(join("unicorn", "**", "bathroom.*")).test(
join("unicorn", "sleeping", "in", "bathroom.py")
),
true
);
assertEquals(
glob(join("unicorn", "!(sleeping)", "bathroom.ts"), {
globToRegExp(join("unicorn", "!(sleeping)", "bathroom.ts"), {
extended: true
}).test(join("unicorn", "flying", "bathroom.ts")),
true
);
assertEquals(
glob(join("unicorn", "(!sleeping)", "bathroom.ts"), {
globToRegExp(join("unicorn", "(!sleeping)", "bathroom.ts"), {
extended: true
}).test(join("unicorn", "sleeping", "bathroom.ts")),
false
......@@ -61,7 +62,7 @@ testWalk(
await touch(d + "/a/x.ts");
},
async function globInWalk(): Promise<void> {
const arr = await walkArray(".", { match: [glob("*.ts")] });
const arr = await walkArray(".", { match: [globToRegExp("*.ts")] });
assertEquals(arr.length, 1);
assertEquals(arr[0], "a/x.ts");
}
......@@ -76,7 +77,7 @@ testWalk(
await touch(d + "/b/z.js");
},
async function globInWalkWildcardFiles(): Promise<void> {
const arr = await walkArray(".", { match: [glob("*.ts")] });
const arr = await walkArray(".", { match: [globToRegExp("*.ts")] });
assertEquals(arr.length, 2);
assertEquals(arr[0], "a/x.ts");
assertEquals(arr[1], "b/z.ts");
......@@ -92,7 +93,7 @@ testWalk(
async function globInWalkFolderWildcard(): Promise<void> {
const arr = await walkArray(".", {
match: [
glob(join("a", "**", "*.ts"), {
globToRegExp(join("a", "**", "*.ts"), {
flags: "g",
globstar: true
})
......@@ -116,7 +117,7 @@ testWalk(
async function globInWalkFolderExtended(): Promise<void> {
const arr = await walkArray(".", {
match: [
glob(join("a", "+(raptor|deno)", "*.ts"), {
globToRegExp(join("a", "+(raptor|deno)", "*.ts"), {
flags: "g",
extended: true
})
......@@ -136,7 +137,7 @@ testWalk(
},
async function globInWalkWildcardExtension(): Promise<void> {
const arr = await walkArray(".", {
match: [glob("x.*", { flags: "g", globstar: true })]
match: [globToRegExp("x.*", { flags: "g", globstar: true })]
});
assertEquals(arr.length, 2);
assertEquals(arr[0], "x.js");
......@@ -259,25 +260,34 @@ test({
}
});
test(function normalizeGlobGlobstar(): void {
assertEquals(normalizeGlob(`**${SEP}..`, { globstar: true }), `**${SEP}..`);
});
test(function joinGlobsGlobstar(): void {
assertEquals(joinGlobs(["**", ".."], { globstar: true }), `**${SEP}..`);
});
async function expandGlobArray(
globString: string,
options: ExpandGlobOptions
): Promise<string[]> {
const infos: WalkInfo[] = [];
for await (const info of expandGlob(globString, options)) {
infos.push(info);
const paths: string[] = [];
for await (const { filename } of expandGlob(globString, options)) {
paths.push(filename);
}
infos.sort();
const infosSync = [...expandGlobSync(globString, options)];
infosSync.sort();
assertEquals(infos, infosSync);
paths.sort();
const pathsSync = [...expandGlobSync(globString, options)].map(
({ filename }): string => filename
);
pathsSync.sort();
assertEquals(paths, pathsSync);
const root = normalize(options.root || cwd());
const paths = infos.map(({ filename }): string => filename);
for (const path of paths) {
assert(path.startsWith(root));
}
const relativePaths = paths.map((path: string): string =>
relative(root, path)
const relativePaths = paths.map(
(path: string): string => relative(root, path) || "."
);
relativePaths.sort();
return relativePaths;
......@@ -288,16 +298,38 @@ function urlToFilePath(url: URL): string {
return url.pathname.slice(url.protocol == "file:" && isWindows ? 1 : 0);
}
const EG_OPTIONS = {
const EG_OPTIONS: ExpandGlobOptions = {
root: urlToFilePath(new URL(join("testdata", "glob"), import.meta.url)),
includeDirs: true,
extended: false,
globstar: false,
strict: false,
filepath: false,
flags: ""
globstar: false
};
test(async function expandGlobWildcard(): Promise<void> {
const options = EG_OPTIONS;
assertEquals(await expandGlobArray("*", options), [
"abc",
"abcdef",
"abcdefghi",
"subdir"
]);
});
test(async function expandGlobTrailingSeparator(): Promise<void> {
const options = EG_OPTIONS;
assertEquals(await expandGlobArray("*/", options), ["subdir"]);
});
test(async function expandGlobParent(): Promise<void> {
const options = EG_OPTIONS;
assertEquals(await expandGlobArray("subdir/../*", options), [
"abc",
"abcdef",
"abcdefghi",
"subdir"
]);
});
test(async function expandGlobExt(): Promise<void> {
const options = { ...EG_OPTIONS, extended: true };
assertEquals(await expandGlobArray("abc?(def|ghi)", options), [
......@@ -320,10 +352,18 @@ test(async function expandGlobExt(): Promise<void> {
test(async function expandGlobGlobstar(): Promise<void> {
const options = { ...EG_OPTIONS, globstar: true };
assertEquals(await expandGlobArray(join("**", "abc"), options), [
"abc",
join("subdir", "abc")
]);
assertEquals(
await expandGlobArray(joinGlobs(["**", "abc"], options), options),
["abc", join("subdir", "abc")]
);
});
test(async function expandGlobGlobstarParent(): Promise<void> {
const options = { ...EG_OPTIONS, globstar: true };
assertEquals(
await expandGlobArray(joinGlobs(["subdir", "**", ".."], options), options),
["."]
);
});
test(async function expandGlobIncludeDirs(): Promise<void> {
......
......@@ -2,8 +2,6 @@
// MIT License
// Copyright (c) 2018 Terkel Gjervig Nielsen
import { GlobOptions } from "./glob.ts";
const isWin = Deno.build.os === "win";
const SEP = isWin ? `(\\\\+|\\/)` : `\\/`;
const SEP_ESC = isWin ? `\\\\` : `/`;
......@@ -13,6 +11,22 @@ const WILDCARD = `([^${SEP_ESC}/]*)`;
const GLOBSTAR_SEGMENT = `((?:[^${SEP_ESC}/]*(?:${SEP_ESC}|\/|$))*)`;
const WILDCARD_SEGMENT = `([^${SEP_ESC}/]*)`;
export interface GlobrexOptions {
// Allow ExtGlob features
extended?: boolean;
// When globstar is true, '/foo/**' is equivelant
// to '/foo/*' when globstar is false.
// Having globstar set to true is the same usage as
// using wildcards in bash
globstar?: boolean;
// be laissez faire about mutiple slashes
strict?: boolean;
// Parse as filepath for extra path related features
filepath?: boolean;
// Flag to use in the generated RegExp
flags?: string;
}
export interface GlobrexResult {
regex: RegExp;
path?: {
......@@ -41,7 +55,7 @@ export function globrex(
strict = false,
filepath = false,
flags = ""
}: GlobOptions = {}
}: GlobrexOptions = {}
): GlobrexResult {
let regex = "";
let segment = "";
......
......@@ -51,3 +51,4 @@ export const CHAR_9 = 57; /* 9 */
export const isWindows = build.os === "win";
export const EOL = isWindows ? "\r\n" : "\n";
export const SEP = isWindows ? "\\" : "/";
export const SEP_PATTERN = isWindows ? /[\\/]+/ : /\/+/;
// Documentation and interface for walk were adapted from Go
// https://golang.org/pkg/path/filepath/#Walk
// Copyright 2009 The Go Authors. All rights reserved. BSD license.
const { readDir, readDirSync, stat, statSync } = Deno;
type FileInfo = Deno.FileInfo;
import { unimplemented } from "../testing/asserts.ts";
import { join } from "./path/mod.ts";
const { readDir, readDirSync, stat, statSync } = Deno;
type FileInfo = Deno.FileInfo;
export interface WalkOptions {
maxDepth?: number;
includeFiles?: boolean;
includeDirs?: boolean;
followSymlinks?: boolean;
exts?: string[];
match?: RegExp[];
skip?: RegExp[];
onError?: (err: Error) => void;
followSymlinks?: boolean;
}
function patternTest(patterns: RegExp[], path: string): boolean {
......@@ -54,13 +55,14 @@ export interface WalkInfo {
* directories walk() can be inefficient.
*
* Options:
* - maxDepth?: number;
* - includeDirs?: boolean;
* - maxDepth?: number = Infinity;
* - includeFiles?: boolean = true;
* - includeDirs?: boolean = true;
* - followSymlinks?: boolean = false;
* - exts?: string[];
* - match?: RegExp[];
* - skip?: RegExp[];
* - onError?: (err: Error) => void;
* - followSymlinks?: boolean;
*
* for await (const { filename, info } of walk(".")) {
* console.log(filename);
......@@ -71,10 +73,24 @@ export async function* walk(
root: string,
options: WalkOptions = {}
): AsyncIterableIterator<WalkInfo> {
options.maxDepth! -= 1;
if (options.includeDirs && include(root, options)) {
const rootInfo = await stat(root);
yield { filename: root, info: rootInfo };
const maxDepth = options.maxDepth != undefined ? options.maxDepth! : Infinity;
if (maxDepth < 0) {
return;
}
if (options.includeDirs != false && include(root, options)) {
let rootInfo: FileInfo;
try {
rootInfo = await stat(root);
} catch (err) {
if (options.onError) {
options.onError(err);
return;
}
}
yield { filename: root, info: rootInfo! };
}
if (maxDepth < 1 || patternTest(options.skip || [], root)) {
return;
}
let ls: FileInfo[] = [];
try {
......@@ -97,26 +113,38 @@ export async function* walk(
const filename = join(root, info.name!);
if (info.isFile()) {
if (include(filename, options)) {
if (options.includeFiles != false && include(filename, options)) {
yield { filename, info };
}
} else {
if (!(options.maxDepth! < 0)) {
yield* walk(filename, options);
}
yield* walk(filename, { ...options, maxDepth: maxDepth - 1 });
}
}
}
/** Same as walk() but uses synchronous ops */
export function* walkSync(
root: string = ".",
root: string,
options: WalkOptions = {}
): IterableIterator<WalkInfo> {
options.maxDepth! -= 1;
if (options.includeDirs && include(root, options)) {
const rootInfo = statSync(root);
yield { filename: root, info: rootInfo };
const maxDepth = options.maxDepth != undefined ? options.maxDepth! : Infinity;
if (maxDepth < 0) {
return;
}
if (options.includeDirs != false && include(root, options)) {
let rootInfo: FileInfo;
try {
rootInfo = statSync(root);
} catch (err) {
if (options.onError) {
options.onError(err);
return;
}
}
yield { filename: root, info: rootInfo! };
}
if (maxDepth < 1 || patternTest(options.skip || [], root)) {
return;
}
let ls: FileInfo[] = [];
try {
......@@ -138,13 +166,11 @@ export function* walkSync(
const filename = join(root, info.name!);
if (info.isFile()) {
if (include(filename, options)) {
if (options.includeFiles != false && include(filename, options)) {
yield { filename, info };
}
} else {
if (!(options.maxDepth! < 0)) {
yield* walkSync(filename, options);
}
yield* walkSync(filename, { ...options, maxDepth: maxDepth - 1 });
}
}
}
const { cwd, chdir, makeTempDir, mkdir, open, remove } = Deno;
type FileInfo = Deno.FileInfo;
import { walk, walkSync, WalkOptions, WalkInfo } from "./walk.ts";
import { test, TestFunction, runIfMain } from "../testing/mod.ts";
import { assertEquals } from "../testing/asserts.ts";
......@@ -29,7 +28,7 @@ function normalize({ filename }: WalkInfo): string {
}
export async function walkArray(
root: string = ".",
root: string,
options: WalkOptions = {}
): Promise<string[]> {
const arr: string[] = [];
......@@ -48,7 +47,7 @@ export async function touch(path: string): Promise<void> {
}
function assertReady(expectedLength: number): void {
const arr = Array.from(walkSync(), normalize);
const arr = Array.from(walkSync("."), normalize);
assertEquals(arr.length, expectedLength);
}
......@@ -58,8 +57,8 @@ testWalk(
await mkdir(d + "/empty");
},
async function emptyDir(): Promise<void> {
const arr = await walkArray();
assertEquals(arr.length, 0);
const arr = await walkArray(".");
assertEquals(arr, [".", "empty"]);
}
);
......@@ -68,9 +67,8 @@ testWalk(
await touch(d + "/x");
},
async function singleFile(): Promise<void> {
const arr = await walkArray();
assertEquals(arr.length, 1);
assertEquals(arr[0], "x");
const arr = await walkArray(".");
assertEquals(arr, [".", "x"]);
}
);
......@@ -83,11 +81,11 @@ testWalk(
for (const _ of walkSync(".")) {
count += 1;
}
assertEquals(count, 1);
assertEquals(count, 2);
for await (const _ of walk(".")) {
count += 1;
}
assertEquals(count, 2);
assertEquals(count, 4);
}
);
......@@ -97,9 +95,8 @@ testWalk(
await touch(d + "/a/x");
},
async function nestedSingleFile(): Promise<void> {
const arr = await walkArray();
assertEquals(arr.length, 1);
assertEquals(arr[0], "a/x");
const arr = await walkArray(".");
assertEquals(arr, [".", "a", "a/x"]);
}
);
......@@ -109,12 +106,11 @@ testWalk(
await touch(d + "/a/b/c/d/x");
},
async function depth(): Promise<void> {
assertReady(1);
assertReady(6);
const arr3 = await walkArray(".", { maxDepth: 3 });
assertEquals(arr3.length, 0);
assertEquals(arr3, [".", "a", "a/b", "a/b/c"]);
const arr5 = await walkArray(".", { maxDepth: 5 });
assertEquals(arr5.length, 1);
assertEquals(arr5[0], "a/b/c/d/x");
assertEquals(arr5, [".", "a", "a/b", "a/b/c", "a/b/c/d", "a/b/c/d/x"]);
}
);
......@@ -125,10 +121,22 @@ testWalk(
await touch(d + "/b/c");
},
async function includeDirs(): Promise<void> {
assertReady(2);
const arr = await walkArray(".", { includeDirs: true });
assertEquals(arr.length, 4);
assertEquals(arr, [".", "a", "b", "b/c"]);
assertReady(4);
const arr = await walkArray(".", { includeDirs: false });
assertEquals(arr, ["a", "b/c"]);
}
);
testWalk(
async (d: string): Promise<void> => {
await touch(d + "/a");
await mkdir(d + "/b");
await touch(d + "/b/c");
},
async function includeFiles(): Promise<void> {
assertReady(4);
const arr = await walkArray(".", { includeFiles: false });
assertEquals(arr, [".", "b"]);
}
);
......@@ -138,10 +146,9 @@ testWalk(
await touch(d + "/y.rs");
},
async function ext(): Promise<void> {
assertReady(2);
assertReady(3);
const arr = await walkArray(".", { exts: [".ts"] });
assertEquals(arr.length, 1);
assertEquals(arr[0], "x.ts");
assertEquals(arr, ["x.ts"]);
}
);
......@@ -152,11 +159,9 @@ testWalk(
await touch(d + "/z.py");
},
async function extAny(): Promise<void> {
assertReady(3);
assertReady(4);
const arr = await walkArray(".", { exts: [".rs", ".ts"] });
assertEquals(arr.length, 2);
assertEquals(arr[0], "x.ts");
assertEquals(arr[1], "y.rs");
assertEquals(arr, ["x.ts", "y.rs"]);
}
);
......@@ -166,10 +171,9 @@ testWalk(
await touch(d + "/y");
},
async function match(): Promise<void> {
assertReady(2);
assertReady(3);
const arr = await walkArray(".", { match: [/x/] });
assertEquals(arr.length, 1);
assertEquals(arr[0], "x");
assertEquals(arr, ["x"]);
}
);
......@@ -180,11 +184,9 @@ testWalk(
await touch(d + "/z");
},
async function matchAny(): Promise<void> {
assertReady(3);
assertReady(4);
const arr = await walkArray(".", { match: [/x/, /y/] });
assertEquals(arr.length, 2);
assertEquals(arr[0], "x");
assertEquals(arr[1], "y");
assertEquals(arr, ["x", "y"]);
}
);
......@@ -194,10 +196,9 @@ testWalk(
await touch(d + "/y");
},
async function skip(): Promise<void> {
assertReady(2);
assertReady(3);
const arr = await walkArray(".", { skip: [/x/] });
assertEquals(arr.length, 1);
assertEquals(arr[0], "y");
assertEquals(arr, [".", "y"]);
}
);
......@@ -208,10 +209,9 @@ testWalk(
await touch(d + "/z");
},
async function skipAny(): Promise<void> {
assertReady(3);
assertReady(4);
const arr = await walkArray(".", { skip: [/x/, /y/] });
assertEquals(arr.length, 1);
assertEquals(arr[0], "z");
assertEquals(arr, [".", "z"]);
}
);
......@@ -224,19 +224,18 @@ testWalk(
await touch(d + "/b/z");
},
async function subDir(): Promise<void> {
assertReady(3);
assertReady(6);
const arr = await walkArray("b");
assertEquals(arr.length, 1);
assertEquals(arr[0], "b/z");
assertEquals(arr, ["b", "b/z"]);
}
);
testWalk(
async (_d: string): Promise<void> => {},
async function onError(): Promise<void> {
assertReady(0);
assertReady(1);
const ignored = await walkArray("missing");
assertEquals(ignored.length, 0);
assertEquals(ignored, ["missing"]);
let errors = 0;
await walkArray("missing", { onError: (_e): number => (errors += 1) });
// It's 2 since walkArray iterates over both sync and async.
......@@ -265,7 +264,7 @@ testWalk(
return;
}
assertReady(3);
assertReady(6);
const files = await walkArray("a");
assertEquals(files.length, 2);
assert(!files.includes("a/bb/z"));
......
......@@ -23,11 +23,10 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
// This script formats the given source files. If the files are omitted, it
// formats the all files in the repository.
const { args, exit, readFile, writeFile, stdout, stdin, readAll } = Deno;
import { glob, isGlob, GlobOptions } from "../fs/glob.ts";
import { walk, WalkInfo } from "../fs/walk.ts";
import { parse } from "../flags/mod.ts";
import { ExpandGlobOptions, WalkInfo, expandGlob } from "../fs/mod.ts";
import { prettier, prettierPlugins } from "./prettier.ts";
const { args, cwd, exit, readAll, readFile, stdin, stdout, writeFile } = Deno;
const HELP_MESSAGE = `
Formats the given files. If no arg is passed, then formats the all files.
......@@ -289,65 +288,47 @@ async function formatFromStdin(
/**
* Get the files to format.
* @param selectors The glob patterns to select the files.
* eg `cmd/*.ts` to select all the typescript files in cmd
* @param include The glob patterns to select the files.
* eg `"cmd/*.ts"` to select all the typescript files in cmd
* directory.
* eg `cmd/run.ts` to select `cmd/run.ts` file as only.
* @param ignore The glob patterns to ignore files.
* eg `*_test.ts` to ignore all the test file.
* @param options options to pass to `glob(selector, options)`
* eg `"cmd/run.ts"` to select `cmd/run.ts` file as only.
* @param exclude The glob patterns to ignore files.
* eg `"*_test.ts"` to ignore all the test file.
* @param root The directory from which to apply default globs.
* @returns returns an async iterable object
*/
function getTargetFiles(
selectors: string[],
ignore: string[] = [],
options: GlobOptions = {}
async function* getTargetFiles(
include: string[],
exclude: string[],
root: string = cwd()
): AsyncIterableIterator<WalkInfo> {
const matchers: Array<string | RegExp> = [];
const selectorMap: { [k: string]: boolean } = {};
const expandGlobOpts: ExpandGlobOptions = {
root,
exclude,
includeDirs: true,
extended: true,
globstar: true
};
for (const selector of selectors) {
// If the selector already exists then ignore it.
if (selectorMap[selector]) continue;
selectorMap[selector] = true;
if (isGlob(selector) || selector === ".") {
matchers.push(glob(selector, options));
} else {
matchers.push(selector);
async function* expandDirectory(d: string): AsyncIterableIterator<WalkInfo> {
for await (const walkInfo of expandGlob("**/*", {
...expandGlobOpts,
root: d,
includeDirs: false
})) {
yield walkInfo;
}
}
const skip = ignore.map((i: string): RegExp => glob(i, options));
return (async function*(): AsyncIterableIterator<WalkInfo> {
for (const match of matchers) {
if (typeof match === "string") {
const fileInfo = await Deno.stat(match);
if (fileInfo.isDirectory()) {
const files = walk(match, { skip });
for await (const info of files) {
yield info;
}
} else {
const info: WalkInfo = {
filename: match,
info: fileInfo
};
yield info;
}
for (const globString of include) {
for await (const walkInfo of expandGlob(globString, expandGlobOpts)) {
if (walkInfo.info.isDirectory()) {
yield* expandDirectory(walkInfo.filename);
} else {
const files = walk(".", { match: [match], skip });
for await (const info of files) {
yield info;
}
yield walkInfo;
}
}
})();
}
}
async function main(opts): Promise<void> {
......@@ -371,12 +352,10 @@ async function main(opts): Promise<void> {
console.log(HELP_MESSAGE);
exit(0);
}
const options: GlobOptions = { flags: "g" };
const files = getTargetFiles(
args.length ? args : ["."],
Array.isArray(ignore) ? ignore : [ignore],
options
Array.isArray(ignore) ? ignore : [ignore]
);
const tty = Deno.isTTY();
......
#!/usr/bin/env -S deno -A
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import { parse } from "../flags/mod.ts";
import {
WalkInfo,
expandGlobSync,
glob,
ExpandGlobOptions
} from "../fs/mod.ts";
import { ExpandGlobOptions, expandGlob } from "../fs/mod.ts";
import { isWindows } from "../fs/path/constants.ts";
import { isAbsolute, join } from "../fs/path/mod.ts";
import { join } from "../fs/path/mod.ts";
import { RunTestsOptions, runTests } from "./mod.ts";
const { DenoError, ErrorKind, args, cwd, exit } = Deno;
......@@ -64,60 +59,56 @@ function filePathToUrl(path: string): string {
return `file://${isWindows ? "/" : ""}${path.replace(/\\/g, "/")}`;
}
function expandDirectory(dir: string, options: ExpandGlobOptions): WalkInfo[] {
return DIR_GLOBS.flatMap((s: string): WalkInfo[] => [
...expandGlobSync(s, { ...options, root: dir })
]);
}
/**
* Given a list of globs or URLs to include and exclude and a root directory
* from which to expand relative globs, return a list of URLs
* from which to expand relative globs, yield a list of URLs
* (file: or remote) that should be imported for the test runner.
*/
export async function findTestModules(
export async function* findTestModules(
includeModules: string[],
excludeModules: string[],
root: string = cwd()
): Promise<string[]> {
): AsyncIterableIterator<string> {
const [includePaths, includeUrls] = partition(includeModules, isRemoteUrl);
const [excludePaths, excludeUrls] = partition(excludeModules, isRemoteUrl);
const expandGlobOpts = {
const expandGlobOpts: ExpandGlobOptions = {
root,
exclude: excludePaths,
includeDirs: true,
extended: true,
globstar: true,
filepath: true
globstar: true
};
// TODO: We use the `g` flag here to support path prefixes when specifying
// excludes. Replace with a solution that does this more correctly.
const excludePathPatterns = excludePaths.map(
(s: string): RegExp =>
glob(isAbsolute(s) ? s : join(root, s), { ...expandGlobOpts, flags: "g" })
);
async function* expandDirectory(d: string): AsyncIterableIterator<string> {
for (const dirGlob of DIR_GLOBS) {
for await (const walkInfo of expandGlob(dirGlob, {
...expandGlobOpts,
root: d,
includeDirs: false
})) {
yield filePathToUrl(walkInfo.filename);
}
}
}
for (const globString of includePaths) {
for await (const walkInfo of expandGlob(globString, expandGlobOpts)) {
if (walkInfo.info.isDirectory()) {
yield* expandDirectory(walkInfo.filename);
} else {
yield filePathToUrl(walkInfo.filename);
}
}
}
const excludeUrlPatterns = excludeUrls.map(
(url: string): RegExp => RegExp(url)
);
const notExcludedPath = ({ filename }: WalkInfo): boolean =>
!excludePathPatterns.some((p: RegExp): boolean => !!filename.match(p));
const notExcludedUrl = (url: string): boolean =>
const shouldIncludeUrl = (url: string): boolean =>
!excludeUrlPatterns.some((p: RegExp): boolean => !!url.match(p));
const matchedPaths = includePaths
.flatMap((s: string): WalkInfo[] => [...expandGlobSync(s, expandGlobOpts)])
.filter(notExcludedPath)
.flatMap(({ filename, info }): string[] =>
info.isDirectory()
? expandDirectory(filename, { ...expandGlobOpts, includeDirs: false })
.filter(notExcludedPath)
.map(({ filename }): string => filename)
: [filename]
);
const matchedUrls = includeUrls.filter(notExcludedUrl);
return [...matchedPaths.map(filePathToUrl), ...matchedUrls];
yield* includeUrls.filter(shouldIncludeUrl);
}
export interface RunTestModulesOptions extends RunTestsOptions {
......@@ -162,9 +153,13 @@ export async function runTestModules({
skip = /^\s*$/,
disableLog = false
}: RunTestModulesOptions = {}): Promise<void> {
const testModuleUrls = await findTestModules(include, exclude);
let moduleCount = 0;
for await (const testModule of findTestModules(include, exclude)) {
await import(testModule);
moduleCount++;
}
if (testModuleUrls.length == 0) {
if (moduleCount == 0) {
const noneFoundMessage = "No matching test modules found.";
if (!allowNone) {
throw new DenoError(ErrorKind.NotFound, noneFoundMessage);
......@@ -175,11 +170,7 @@ export async function runTestModules({
}
if (!disableLog) {
console.log(`Found ${testModuleUrls.length} matching test modules.`);
}
for (const url of testModuleUrls) {
await import(url);
console.log(`Found ${moduleCount} matching test modules.`);
}
await runTests({
......
......@@ -3,17 +3,30 @@ import { test } from "./mod.ts";
import { findTestModules } from "./runner.ts";
import { isWindows } from "../fs/path/constants.ts";
import { assertEquals } from "../testing/asserts.ts";
const { cwd } = Deno;
function urlToFilePath(url: URL): string {
// Since `new URL('file:///C:/a').pathname` is `/C:/a`, remove leading slash.
return url.pathname.slice(url.protocol == "file:" && isWindows ? 1 : 0);
}
async function findTestModulesArray(
include: string[],
exclude: string[],
root: string = cwd()
): Promise<string[]> {
const result = [];
for await (const testModule of findTestModules(include, exclude, root)) {
result.push(testModule);
}
return result;
}
const TEST_DATA_URL = new URL("testdata", import.meta.url);
const TEST_DATA_PATH = urlToFilePath(TEST_DATA_URL);
test(async function findTestModulesDir1(): Promise<void> {
const urls = await findTestModules(["."], [], TEST_DATA_PATH);
const urls = await findTestModulesArray(["."], [], TEST_DATA_PATH);
assertEquals(urls.sort(), [
`${TEST_DATA_URL}/bar_test.js`,
`${TEST_DATA_URL}/foo_test.ts`,
......@@ -27,7 +40,7 @@ test(async function findTestModulesDir1(): Promise<void> {
});
test(async function findTestModulesDir2(): Promise<void> {
const urls = await findTestModules(["subdir"], [], TEST_DATA_PATH);
const urls = await findTestModulesArray(["subdir"], [], TEST_DATA_PATH);
assertEquals(urls.sort(), [
`${TEST_DATA_URL}/subdir/bar_test.js`,
`${TEST_DATA_URL}/subdir/foo_test.ts`,
......@@ -37,7 +50,11 @@ test(async function findTestModulesDir2(): Promise<void> {
});
test(async function findTestModulesGlob(): Promise<void> {
const urls = await findTestModules(["**/*_test.{js,ts}"], [], TEST_DATA_PATH);
const urls = await findTestModulesArray(
["**/*_test.{js,ts}"],
[],
TEST_DATA_PATH
);
assertEquals(urls.sort(), [
`${TEST_DATA_URL}/bar_test.js`,
`${TEST_DATA_URL}/foo_test.ts`,
......@@ -47,7 +64,7 @@ test(async function findTestModulesGlob(): Promise<void> {
});
test(async function findTestModulesExcludeDir(): Promise<void> {
const urls = await findTestModules(["."], ["subdir"], TEST_DATA_PATH);
const urls = await findTestModulesArray(["."], ["subdir"], TEST_DATA_PATH);
assertEquals(urls.sort(), [
`${TEST_DATA_URL}/bar_test.js`,
`${TEST_DATA_URL}/foo_test.ts`,
......@@ -57,7 +74,7 @@ test(async function findTestModulesExcludeDir(): Promise<void> {
});
test(async function findTestModulesExcludeGlob(): Promise<void> {
const urls = await findTestModules(["."], ["**/foo*"], TEST_DATA_PATH);
const urls = await findTestModulesArray(["."], ["**/foo*"], TEST_DATA_PATH);
assertEquals(urls.sort(), [
`${TEST_DATA_URL}/bar_test.js`,
`${TEST_DATA_URL}/subdir/bar_test.js`,
......@@ -73,6 +90,6 @@ test(async function findTestModulesRemote(): Promise<void> {
"https://example.com/colors_test.ts",
"http://example.com/printf_test.ts"
];
const matches = await findTestModules(urls, []);
const matches = await findTestModulesArray(urls, []);
assertEquals(matches, urls);
});
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册