diff --git a/format.ts b/format.ts index 08d1aaed9fa51118b99430af32796079665c987f..0b44ef4dfe3d9b7a4dfcbde8170e464420cbb2fd 100755 --- a/format.ts +++ b/format.ts @@ -14,9 +14,9 @@ async function main(opts): Promise { "--ignore", "node_modules", "--ignore", - "testdata", + "**/testdata", "--ignore", - "vendor", + "**/vendor", "--write" ]; diff --git a/fs/README.md b/fs/README.md index deb0f5c1abc841009a01c974767b9622a36bb478..b774e061a66499c954b948df9078b6cd13eb6edc 100644 --- a/fs/README.md +++ b/fs/README.md @@ -100,16 +100,16 @@ exists("./foo"); // returns a Promise existsSync("./foo"); // returns boolean ``` -### glob +### globToRegExp Generate a regex based on glob pattern and options This was meant to be using the the `fs.walk` function but can be used anywhere else. ```ts -import { glob } from "https://deno.land/std/fs/mod.ts"; +import { globToRegExp } from "https://deno.land/std/fs/mod.ts"; -glob("foo/**/*.json", { +globToRegExp("foo/**/*.json", { flags: "g", extended: true, globstar: true @@ -160,7 +160,7 @@ Iterate all files in a directory recursively. ```ts import { walk, walkSync } from "https://deno.land/std/fs/mod.ts"; -for (const fileInfo of walkSync()) { +for (const fileInfo of walkSync(".")) { console.log(fileInfo.filename); } diff --git a/fs/glob.ts b/fs/glob.ts index b974f2b29f2e977d3b308cff2b00cbb6aec53ebe..d9e6560c1148e1ac5974c5c69715fc3d46c47ce9 100644 --- a/fs/glob.ts +++ b/fs/glob.ts @@ -1,21 +1,16 @@ import { globrex } from "./globrex.ts"; -import { isAbsolute, join } from "./path/mod.ts"; +import { SEP, SEP_PATTERN, isWindows } from "./path/constants.ts"; +import { isAbsolute, join, normalize } from "./path/mod.ts"; import { WalkInfo, walk, walkSync } from "./walk.ts"; -const { cwd } = Deno; +const { DenoError, ErrorKind, cwd, stat, statSync } = Deno; +type FileInfo = Deno.FileInfo; export interface GlobOptions { - // Allow ExtGlob features extended?: boolean; - // When globstar is true, '/foo/**' is equivelant - // to '/foo/*' when globstar is false. - // Having globstar set to true is the same usage as - // using wildcards in bash globstar?: boolean; - // be laissez faire about mutiple slashes - strict?: boolean; - // Parse as filepath for extra path related features - filepath?: boolean; - // Flag to use in the generated RegExp +} + +export interface GlobToRegExpOptions extends GlobOptions { flags?: string; } @@ -27,12 +22,12 @@ export interface GlobOptions { * * Looking for all the `ts` files: * walkSync(".", { - * match: [glob("*.ts")] + * match: [globToRegExp("*.ts")] * }) * * Looking for all the `.json` files in any subfolder: * walkSync(".", { - * match: [glob(join("a", "**", "*.json"),{ + * match: [globToRegExp(join("a", "**", "*.json"),{ * flags: "g", * extended: true, * globstar: true @@ -43,12 +38,12 @@ export interface GlobOptions { * @param options - Specific options for the glob pattern * @returns A RegExp for the glob pattern */ -export function glob(glob: string, options: GlobOptions = {}): RegExp { - const result = globrex(glob, options); - if (options.filepath) { - return result.path!.regex; - } - return result.regex; +export function globToRegExp( + glob: string, + options: GlobToRegExpOptions = {} +): RegExp { + const result = globrex(glob, { ...options, strict: false, filepath: true }); + return result.path!.regex; } /** Test whether the given string is a glob */ @@ -84,11 +79,78 @@ export function isGlob(str: string): boolean { return false; } +/** Like normalize(), but doesn't collapse "**\/.." when `globstar` is true. */ +export function normalizeGlob( + glob: string, + { globstar = false }: GlobOptions = {} +): string { + if (!!glob.match(/\0/g)) { + throw new DenoError( + ErrorKind.InvalidPath, + `Glob contains invalid characters: "${glob}"` + ); + } + if (!globstar) { + return normalize(glob); + } + const s = SEP_PATTERN.source; + const badParentPattern = new RegExp( + `(?<=(${s}|^)\\*\\*${s})\\.\\.(?=${s}|$)`, + "g" + ); + return normalize(glob.replace(badParentPattern, "\0")).replace(/\0/g, ".."); +} + +/** Like join(), but doesn't collapse "**\/.." when `globstar` is true. */ +export function joinGlobs( + globs: string[], + { extended = false, globstar = false }: GlobOptions = {} +): string { + if (!globstar || globs.length == 0) { + return join(...globs); + } + if (globs.length === 0) return "."; + let joined: string | undefined; + for (const glob of globs) { + let path = glob; + if (path.length > 0) { + if (!joined) joined = path; + else joined += `${SEP}${path}`; + } + } + if (!joined) return "."; + return normalizeGlob(joined, { extended, globstar }); +} + export interface ExpandGlobOptions extends GlobOptions { root?: string; + exclude?: string[]; includeDirs?: boolean; } +interface SplitPath { + segments: string[]; + isAbsolute: boolean; + hasTrailingSep: boolean; + // Defined for any absolute Windows path. + winRoot?: string; +} + +// TODO: Maybe make this public somewhere. +function split(path: string): SplitPath { + const s = SEP_PATTERN.source; + const segments = path + .replace(new RegExp(`^${s}|${s}$`, "g"), "") + .split(SEP_PATTERN); + const isAbsolute_ = isAbsolute(path); + return { + segments, + isAbsolute: isAbsolute_, + hasTrailingSep: !!path.match(new RegExp(`${s}$`)), + winRoot: isWindows && isAbsolute_ ? segments.shift() : undefined + }; +} + /** * Expand the glob string from the specified `root` directory and yield each * result as a `WalkInfo` object. @@ -97,47 +159,203 @@ export interface ExpandGlobOptions extends GlobOptions { // This is a very incomplete solution. The whole directory tree from `root` is // walked and parent paths are not supported. export async function* expandGlob( - globString: string, + glob: string, { root = cwd(), + exclude = [], includeDirs = true, extended = false, - globstar = false, - strict = false, - filepath = true, - flags = "" + globstar = false }: ExpandGlobOptions = {} ): AsyncIterableIterator { - const absoluteGlob = isAbsolute(globString) - ? globString - : join(root, globString); - const globOptions = { extended, globstar, strict, filepath, flags }; - yield* walk(root, { - match: [glob(absoluteGlob, globOptions)], - includeDirs - }); + const globOptions: GlobOptions = { extended, globstar }; + const absRoot = isAbsolute(root) + ? normalize(root) + : joinGlobs([cwd(), root], globOptions); + const resolveFromRoot = (path: string): string => + isAbsolute(path) + ? normalize(path) + : joinGlobs([absRoot, path], globOptions); + const excludePatterns = exclude + .map(resolveFromRoot) + .map((s: string): RegExp => globToRegExp(s, globOptions)); + const shouldInclude = ({ filename }: WalkInfo): boolean => + !excludePatterns.some((p: RegExp): boolean => !!filename.match(p)); + const { segments, hasTrailingSep, winRoot } = split(resolveFromRoot(glob)); + + let fixedRoot = winRoot != undefined ? winRoot : "/"; + while (segments.length > 0 && !isGlob(segments[0])) { + fixedRoot = joinGlobs([fixedRoot, segments.shift()!], globOptions); + } + + let fixedRootInfo: WalkInfo; + try { + fixedRootInfo = { filename: fixedRoot, info: await stat(fixedRoot) }; + } catch { + return; + } + + async function* advanceMatch( + walkInfo: WalkInfo, + globSegment: string + ): AsyncIterableIterator { + if (!walkInfo.info.isDirectory()) { + return; + } else if (globSegment == "..") { + const parentPath = joinGlobs([walkInfo.filename, ".."], globOptions); + try { + return yield* [ + { filename: parentPath, info: await stat(parentPath) } + ].filter(shouldInclude); + } catch { + return; + } + } else if (globSegment == "**") { + return yield* walk(walkInfo.filename, { + includeFiles: false, + skip: excludePatterns + }); + } + yield* walk(walkInfo.filename, { + maxDepth: 1, + match: [ + globToRegExp( + joinGlobs([walkInfo.filename, globSegment], globOptions), + globOptions + ) + ], + skip: excludePatterns + }); + } + + let currentMatches: WalkInfo[] = [fixedRootInfo]; + for (const segment of segments) { + // Advancing the list of current matches may introduce duplicates, so we + // pass everything through this Map. + const nextMatchMap: Map = new Map(); + for (const currentMatch of currentMatches) { + for await (const nextMatch of advanceMatch(currentMatch, segment)) { + nextMatchMap.set(nextMatch.filename, nextMatch.info); + } + } + currentMatches = [...nextMatchMap].sort().map( + ([filename, info]): WalkInfo => ({ + filename, + info + }) + ); + } + if (hasTrailingSep) { + currentMatches = currentMatches.filter(({ info }): boolean => + info.isDirectory() + ); + } + if (!includeDirs) { + currentMatches = currentMatches.filter( + ({ info }): boolean => !info.isDirectory() + ); + } + yield* currentMatches; } /** Synchronous version of `expandGlob()`. */ // TODO: As `expandGlob()`. export function* expandGlobSync( - globString: string, + glob: string, { root = cwd(), + exclude = [], includeDirs = true, extended = false, - globstar = false, - strict = false, - filepath = true, - flags = "" + globstar = false }: ExpandGlobOptions = {} ): IterableIterator { - const absoluteGlob = isAbsolute(globString) - ? globString - : join(root, globString); - const globOptions = { extended, globstar, strict, filepath, flags }; - yield* walkSync(root, { - match: [glob(absoluteGlob, globOptions)], - includeDirs - }); + const globOptions: GlobOptions = { extended, globstar }; + const absRoot = isAbsolute(root) + ? normalize(root) + : joinGlobs([cwd(), root], globOptions); + const resolveFromRoot = (path: string): string => + isAbsolute(path) + ? normalize(path) + : joinGlobs([absRoot, path], globOptions); + const excludePatterns = exclude + .map(resolveFromRoot) + .map((s: string): RegExp => globToRegExp(s, globOptions)); + const shouldInclude = ({ filename }: WalkInfo): boolean => + !excludePatterns.some((p: RegExp): boolean => !!filename.match(p)); + const { segments, hasTrailingSep, winRoot } = split(resolveFromRoot(glob)); + + let fixedRoot = winRoot != undefined ? winRoot : "/"; + while (segments.length > 0 && !isGlob(segments[0])) { + fixedRoot = joinGlobs([fixedRoot, segments.shift()!], globOptions); + } + + let fixedRootInfo: WalkInfo; + try { + fixedRootInfo = { filename: fixedRoot, info: statSync(fixedRoot) }; + } catch { + return; + } + + function* advanceMatch( + walkInfo: WalkInfo, + globSegment: string + ): IterableIterator { + if (!walkInfo.info.isDirectory()) { + return; + } else if (globSegment == "..") { + const parentPath = joinGlobs([walkInfo.filename, ".."], globOptions); + try { + return yield* [ + { filename: parentPath, info: statSync(parentPath) } + ].filter(shouldInclude); + } catch { + return; + } + } else if (globSegment == "**") { + return yield* walkSync(walkInfo.filename, { + includeFiles: false, + skip: excludePatterns + }); + } + yield* walkSync(walkInfo.filename, { + maxDepth: 1, + match: [ + globToRegExp( + joinGlobs([walkInfo.filename, globSegment], globOptions), + globOptions + ) + ], + skip: excludePatterns + }); + } + + let currentMatches: WalkInfo[] = [fixedRootInfo]; + for (const segment of segments) { + // Advancing the list of current matches may introduce duplicates, so we + // pass everything through this Map. + const nextMatchMap: Map = new Map(); + for (const currentMatch of currentMatches) { + for (const nextMatch of advanceMatch(currentMatch, segment)) { + nextMatchMap.set(nextMatch.filename, nextMatch.info); + } + } + currentMatches = [...nextMatchMap].sort().map( + ([filename, info]): WalkInfo => ({ + filename, + info + }) + ); + } + if (hasTrailingSep) { + currentMatches = currentMatches.filter(({ info }): boolean => + info.isDirectory() + ); + } + if (!includeDirs) { + currentMatches = currentMatches.filter( + ({ info }): boolean => !info.isDirectory() + ); + } + yield* currentMatches; } diff --git a/fs/glob_test.ts b/fs/glob_test.ts index 0d180eafd54b0827e2a909642ba13ccd0154238c..df819d92c68f9854432d6716a49c7133f2fa2a71 100644 --- a/fs/glob_test.ts +++ b/fs/glob_test.ts @@ -1,53 +1,54 @@ const { cwd, mkdir } = Deno; import { test, runIfMain } from "../testing/mod.ts"; import { assert, assertEquals } from "../testing/asserts.ts"; -import { isWindows } from "./path/constants.ts"; +import { SEP, isWindows } from "./path/constants.ts"; import { ExpandGlobOptions, expandGlob, - glob, + expandGlobSync, + globToRegExp, isGlob, - expandGlobSync + joinGlobs, + normalizeGlob } from "./glob.ts"; import { join, normalize, relative } from "./path.ts"; -import { WalkInfo } from "./walk.ts"; import { testWalk } from "./walk_test.ts"; import { touch, walkArray } from "./walk_test.ts"; test({ name: "glob: glob to regex", fn(): void { - assertEquals(glob("unicorn.*") instanceof RegExp, true); - assertEquals(glob("unicorn.*").test("poney.ts"), false); - assertEquals(glob("unicorn.*").test("unicorn.py"), true); - assertEquals(glob("*.ts").test("poney.ts"), true); - assertEquals(glob("*.ts").test("unicorn.js"), false); + assertEquals(globToRegExp("unicorn.*") instanceof RegExp, true); + assertEquals(globToRegExp("unicorn.*").test("poney.ts"), false); + assertEquals(globToRegExp("unicorn.*").test("unicorn.py"), true); + assertEquals(globToRegExp("*.ts").test("poney.ts"), true); + assertEquals(globToRegExp("*.ts").test("unicorn.js"), false); assertEquals( - glob(join("unicorn", "**", "cathedral.ts")).test( + globToRegExp(join("unicorn", "**", "cathedral.ts")).test( join("unicorn", "in", "the", "cathedral.ts") ), true ); assertEquals( - glob(join("unicorn", "**", "cathedral.ts")).test( + globToRegExp(join("unicorn", "**", "cathedral.ts")).test( join("unicorn", "in", "the", "kitchen.ts") ), false ); assertEquals( - glob(join("unicorn", "**", "bathroom.*")).test( + globToRegExp(join("unicorn", "**", "bathroom.*")).test( join("unicorn", "sleeping", "in", "bathroom.py") ), true ); assertEquals( - glob(join("unicorn", "!(sleeping)", "bathroom.ts"), { + globToRegExp(join("unicorn", "!(sleeping)", "bathroom.ts"), { extended: true }).test(join("unicorn", "flying", "bathroom.ts")), true ); assertEquals( - glob(join("unicorn", "(!sleeping)", "bathroom.ts"), { + globToRegExp(join("unicorn", "(!sleeping)", "bathroom.ts"), { extended: true }).test(join("unicorn", "sleeping", "bathroom.ts")), false @@ -61,7 +62,7 @@ testWalk( await touch(d + "/a/x.ts"); }, async function globInWalk(): Promise { - const arr = await walkArray(".", { match: [glob("*.ts")] }); + const arr = await walkArray(".", { match: [globToRegExp("*.ts")] }); assertEquals(arr.length, 1); assertEquals(arr[0], "a/x.ts"); } @@ -76,7 +77,7 @@ testWalk( await touch(d + "/b/z.js"); }, async function globInWalkWildcardFiles(): Promise { - const arr = await walkArray(".", { match: [glob("*.ts")] }); + const arr = await walkArray(".", { match: [globToRegExp("*.ts")] }); assertEquals(arr.length, 2); assertEquals(arr[0], "a/x.ts"); assertEquals(arr[1], "b/z.ts"); @@ -92,7 +93,7 @@ testWalk( async function globInWalkFolderWildcard(): Promise { const arr = await walkArray(".", { match: [ - glob(join("a", "**", "*.ts"), { + globToRegExp(join("a", "**", "*.ts"), { flags: "g", globstar: true }) @@ -116,7 +117,7 @@ testWalk( async function globInWalkFolderExtended(): Promise { const arr = await walkArray(".", { match: [ - glob(join("a", "+(raptor|deno)", "*.ts"), { + globToRegExp(join("a", "+(raptor|deno)", "*.ts"), { flags: "g", extended: true }) @@ -136,7 +137,7 @@ testWalk( }, async function globInWalkWildcardExtension(): Promise { const arr = await walkArray(".", { - match: [glob("x.*", { flags: "g", globstar: true })] + match: [globToRegExp("x.*", { flags: "g", globstar: true })] }); assertEquals(arr.length, 2); assertEquals(arr[0], "x.js"); @@ -259,25 +260,34 @@ test({ } }); +test(function normalizeGlobGlobstar(): void { + assertEquals(normalizeGlob(`**${SEP}..`, { globstar: true }), `**${SEP}..`); +}); + +test(function joinGlobsGlobstar(): void { + assertEquals(joinGlobs(["**", ".."], { globstar: true }), `**${SEP}..`); +}); + async function expandGlobArray( globString: string, options: ExpandGlobOptions ): Promise { - const infos: WalkInfo[] = []; - for await (const info of expandGlob(globString, options)) { - infos.push(info); + const paths: string[] = []; + for await (const { filename } of expandGlob(globString, options)) { + paths.push(filename); } - infos.sort(); - const infosSync = [...expandGlobSync(globString, options)]; - infosSync.sort(); - assertEquals(infos, infosSync); + paths.sort(); + const pathsSync = [...expandGlobSync(globString, options)].map( + ({ filename }): string => filename + ); + pathsSync.sort(); + assertEquals(paths, pathsSync); const root = normalize(options.root || cwd()); - const paths = infos.map(({ filename }): string => filename); for (const path of paths) { assert(path.startsWith(root)); } - const relativePaths = paths.map((path: string): string => - relative(root, path) + const relativePaths = paths.map( + (path: string): string => relative(root, path) || "." ); relativePaths.sort(); return relativePaths; @@ -288,16 +298,38 @@ function urlToFilePath(url: URL): string { return url.pathname.slice(url.protocol == "file:" && isWindows ? 1 : 0); } -const EG_OPTIONS = { +const EG_OPTIONS: ExpandGlobOptions = { root: urlToFilePath(new URL(join("testdata", "glob"), import.meta.url)), includeDirs: true, extended: false, - globstar: false, - strict: false, - filepath: false, - flags: "" + globstar: false }; +test(async function expandGlobWildcard(): Promise { + const options = EG_OPTIONS; + assertEquals(await expandGlobArray("*", options), [ + "abc", + "abcdef", + "abcdefghi", + "subdir" + ]); +}); + +test(async function expandGlobTrailingSeparator(): Promise { + const options = EG_OPTIONS; + assertEquals(await expandGlobArray("*/", options), ["subdir"]); +}); + +test(async function expandGlobParent(): Promise { + const options = EG_OPTIONS; + assertEquals(await expandGlobArray("subdir/../*", options), [ + "abc", + "abcdef", + "abcdefghi", + "subdir" + ]); +}); + test(async function expandGlobExt(): Promise { const options = { ...EG_OPTIONS, extended: true }; assertEquals(await expandGlobArray("abc?(def|ghi)", options), [ @@ -320,10 +352,18 @@ test(async function expandGlobExt(): Promise { test(async function expandGlobGlobstar(): Promise { const options = { ...EG_OPTIONS, globstar: true }; - assertEquals(await expandGlobArray(join("**", "abc"), options), [ - "abc", - join("subdir", "abc") - ]); + assertEquals( + await expandGlobArray(joinGlobs(["**", "abc"], options), options), + ["abc", join("subdir", "abc")] + ); +}); + +test(async function expandGlobGlobstarParent(): Promise { + const options = { ...EG_OPTIONS, globstar: true }; + assertEquals( + await expandGlobArray(joinGlobs(["subdir", "**", ".."], options), options), + ["."] + ); }); test(async function expandGlobIncludeDirs(): Promise { diff --git a/fs/globrex.ts b/fs/globrex.ts index e382dc82efc38dff7a43f6f5df074fab32cf9e21..03d69fc8bca4128d9f8238b35d10caea1c9c8279 100644 --- a/fs/globrex.ts +++ b/fs/globrex.ts @@ -2,8 +2,6 @@ // MIT License // Copyright (c) 2018 Terkel Gjervig Nielsen -import { GlobOptions } from "./glob.ts"; - const isWin = Deno.build.os === "win"; const SEP = isWin ? `(\\\\+|\\/)` : `\\/`; const SEP_ESC = isWin ? `\\\\` : `/`; @@ -13,6 +11,22 @@ const WILDCARD = `([^${SEP_ESC}/]*)`; const GLOBSTAR_SEGMENT = `((?:[^${SEP_ESC}/]*(?:${SEP_ESC}|\/|$))*)`; const WILDCARD_SEGMENT = `([^${SEP_ESC}/]*)`; +export interface GlobrexOptions { + // Allow ExtGlob features + extended?: boolean; + // When globstar is true, '/foo/**' is equivelant + // to '/foo/*' when globstar is false. + // Having globstar set to true is the same usage as + // using wildcards in bash + globstar?: boolean; + // be laissez faire about mutiple slashes + strict?: boolean; + // Parse as filepath for extra path related features + filepath?: boolean; + // Flag to use in the generated RegExp + flags?: string; +} + export interface GlobrexResult { regex: RegExp; path?: { @@ -41,7 +55,7 @@ export function globrex( strict = false, filepath = false, flags = "" - }: GlobOptions = {} + }: GlobrexOptions = {} ): GlobrexResult { let regex = ""; let segment = ""; diff --git a/fs/path/constants.ts b/fs/path/constants.ts index 55851f8cc3778be29d52c96213472d326a0ec622..1e1eeeb493d454caa16a22c6443974961e126a03 100644 --- a/fs/path/constants.ts +++ b/fs/path/constants.ts @@ -51,3 +51,4 @@ export const CHAR_9 = 57; /* 9 */ export const isWindows = build.os === "win"; export const EOL = isWindows ? "\r\n" : "\n"; export const SEP = isWindows ? "\\" : "/"; +export const SEP_PATTERN = isWindows ? /[\\/]+/ : /\/+/; diff --git a/fs/walk.ts b/fs/walk.ts index d986ef47f5a486be7f043d5e9e7ede73db92d7a7..583b4dd55082dfc03b1e3063128db20ab110c6ef 100644 --- a/fs/walk.ts +++ b/fs/walk.ts @@ -1,19 +1,20 @@ // Documentation and interface for walk were adapted from Go // https://golang.org/pkg/path/filepath/#Walk // Copyright 2009 The Go Authors. All rights reserved. BSD license. -const { readDir, readDirSync, stat, statSync } = Deno; -type FileInfo = Deno.FileInfo; import { unimplemented } from "../testing/asserts.ts"; import { join } from "./path/mod.ts"; +const { readDir, readDirSync, stat, statSync } = Deno; +type FileInfo = Deno.FileInfo; export interface WalkOptions { maxDepth?: number; + includeFiles?: boolean; includeDirs?: boolean; + followSymlinks?: boolean; exts?: string[]; match?: RegExp[]; skip?: RegExp[]; onError?: (err: Error) => void; - followSymlinks?: boolean; } function patternTest(patterns: RegExp[], path: string): boolean { @@ -54,13 +55,14 @@ export interface WalkInfo { * directories walk() can be inefficient. * * Options: - * - maxDepth?: number; - * - includeDirs?: boolean; + * - maxDepth?: number = Infinity; + * - includeFiles?: boolean = true; + * - includeDirs?: boolean = true; + * - followSymlinks?: boolean = false; * - exts?: string[]; * - match?: RegExp[]; * - skip?: RegExp[]; * - onError?: (err: Error) => void; - * - followSymlinks?: boolean; * * for await (const { filename, info } of walk(".")) { * console.log(filename); @@ -71,10 +73,24 @@ export async function* walk( root: string, options: WalkOptions = {} ): AsyncIterableIterator { - options.maxDepth! -= 1; - if (options.includeDirs && include(root, options)) { - const rootInfo = await stat(root); - yield { filename: root, info: rootInfo }; + const maxDepth = options.maxDepth != undefined ? options.maxDepth! : Infinity; + if (maxDepth < 0) { + return; + } + if (options.includeDirs != false && include(root, options)) { + let rootInfo: FileInfo; + try { + rootInfo = await stat(root); + } catch (err) { + if (options.onError) { + options.onError(err); + return; + } + } + yield { filename: root, info: rootInfo! }; + } + if (maxDepth < 1 || patternTest(options.skip || [], root)) { + return; } let ls: FileInfo[] = []; try { @@ -97,26 +113,38 @@ export async function* walk( const filename = join(root, info.name!); if (info.isFile()) { - if (include(filename, options)) { + if (options.includeFiles != false && include(filename, options)) { yield { filename, info }; } } else { - if (!(options.maxDepth! < 0)) { - yield* walk(filename, options); - } + yield* walk(filename, { ...options, maxDepth: maxDepth - 1 }); } } } /** Same as walk() but uses synchronous ops */ export function* walkSync( - root: string = ".", + root: string, options: WalkOptions = {} ): IterableIterator { - options.maxDepth! -= 1; - if (options.includeDirs && include(root, options)) { - const rootInfo = statSync(root); - yield { filename: root, info: rootInfo }; + const maxDepth = options.maxDepth != undefined ? options.maxDepth! : Infinity; + if (maxDepth < 0) { + return; + } + if (options.includeDirs != false && include(root, options)) { + let rootInfo: FileInfo; + try { + rootInfo = statSync(root); + } catch (err) { + if (options.onError) { + options.onError(err); + return; + } + } + yield { filename: root, info: rootInfo! }; + } + if (maxDepth < 1 || patternTest(options.skip || [], root)) { + return; } let ls: FileInfo[] = []; try { @@ -138,13 +166,11 @@ export function* walkSync( const filename = join(root, info.name!); if (info.isFile()) { - if (include(filename, options)) { + if (options.includeFiles != false && include(filename, options)) { yield { filename, info }; } } else { - if (!(options.maxDepth! < 0)) { - yield* walkSync(filename, options); - } + yield* walkSync(filename, { ...options, maxDepth: maxDepth - 1 }); } } } diff --git a/fs/walk_test.ts b/fs/walk_test.ts index 034e327e2e0bb998db50e73113f8500d0d787189..abd5adbcfd2b933de0f4aae384cd78bf49af5687 100644 --- a/fs/walk_test.ts +++ b/fs/walk_test.ts @@ -1,5 +1,4 @@ const { cwd, chdir, makeTempDir, mkdir, open, remove } = Deno; -type FileInfo = Deno.FileInfo; import { walk, walkSync, WalkOptions, WalkInfo } from "./walk.ts"; import { test, TestFunction, runIfMain } from "../testing/mod.ts"; import { assertEquals } from "../testing/asserts.ts"; @@ -29,7 +28,7 @@ function normalize({ filename }: WalkInfo): string { } export async function walkArray( - root: string = ".", + root: string, options: WalkOptions = {} ): Promise { const arr: string[] = []; @@ -48,7 +47,7 @@ export async function touch(path: string): Promise { } function assertReady(expectedLength: number): void { - const arr = Array.from(walkSync(), normalize); + const arr = Array.from(walkSync("."), normalize); assertEquals(arr.length, expectedLength); } @@ -58,8 +57,8 @@ testWalk( await mkdir(d + "/empty"); }, async function emptyDir(): Promise { - const arr = await walkArray(); - assertEquals(arr.length, 0); + const arr = await walkArray("."); + assertEquals(arr, [".", "empty"]); } ); @@ -68,9 +67,8 @@ testWalk( await touch(d + "/x"); }, async function singleFile(): Promise { - const arr = await walkArray(); - assertEquals(arr.length, 1); - assertEquals(arr[0], "x"); + const arr = await walkArray("."); + assertEquals(arr, [".", "x"]); } ); @@ -83,11 +81,11 @@ testWalk( for (const _ of walkSync(".")) { count += 1; } - assertEquals(count, 1); + assertEquals(count, 2); for await (const _ of walk(".")) { count += 1; } - assertEquals(count, 2); + assertEquals(count, 4); } ); @@ -97,9 +95,8 @@ testWalk( await touch(d + "/a/x"); }, async function nestedSingleFile(): Promise { - const arr = await walkArray(); - assertEquals(arr.length, 1); - assertEquals(arr[0], "a/x"); + const arr = await walkArray("."); + assertEquals(arr, [".", "a", "a/x"]); } ); @@ -109,12 +106,11 @@ testWalk( await touch(d + "/a/b/c/d/x"); }, async function depth(): Promise { - assertReady(1); + assertReady(6); const arr3 = await walkArray(".", { maxDepth: 3 }); - assertEquals(arr3.length, 0); + assertEquals(arr3, [".", "a", "a/b", "a/b/c"]); const arr5 = await walkArray(".", { maxDepth: 5 }); - assertEquals(arr5.length, 1); - assertEquals(arr5[0], "a/b/c/d/x"); + assertEquals(arr5, [".", "a", "a/b", "a/b/c", "a/b/c/d", "a/b/c/d/x"]); } ); @@ -125,10 +121,22 @@ testWalk( await touch(d + "/b/c"); }, async function includeDirs(): Promise { - assertReady(2); - const arr = await walkArray(".", { includeDirs: true }); - assertEquals(arr.length, 4); - assertEquals(arr, [".", "a", "b", "b/c"]); + assertReady(4); + const arr = await walkArray(".", { includeDirs: false }); + assertEquals(arr, ["a", "b/c"]); + } +); + +testWalk( + async (d: string): Promise => { + await touch(d + "/a"); + await mkdir(d + "/b"); + await touch(d + "/b/c"); + }, + async function includeFiles(): Promise { + assertReady(4); + const arr = await walkArray(".", { includeFiles: false }); + assertEquals(arr, [".", "b"]); } ); @@ -138,10 +146,9 @@ testWalk( await touch(d + "/y.rs"); }, async function ext(): Promise { - assertReady(2); + assertReady(3); const arr = await walkArray(".", { exts: [".ts"] }); - assertEquals(arr.length, 1); - assertEquals(arr[0], "x.ts"); + assertEquals(arr, ["x.ts"]); } ); @@ -152,11 +159,9 @@ testWalk( await touch(d + "/z.py"); }, async function extAny(): Promise { - assertReady(3); + assertReady(4); const arr = await walkArray(".", { exts: [".rs", ".ts"] }); - assertEquals(arr.length, 2); - assertEquals(arr[0], "x.ts"); - assertEquals(arr[1], "y.rs"); + assertEquals(arr, ["x.ts", "y.rs"]); } ); @@ -166,10 +171,9 @@ testWalk( await touch(d + "/y"); }, async function match(): Promise { - assertReady(2); + assertReady(3); const arr = await walkArray(".", { match: [/x/] }); - assertEquals(arr.length, 1); - assertEquals(arr[0], "x"); + assertEquals(arr, ["x"]); } ); @@ -180,11 +184,9 @@ testWalk( await touch(d + "/z"); }, async function matchAny(): Promise { - assertReady(3); + assertReady(4); const arr = await walkArray(".", { match: [/x/, /y/] }); - assertEquals(arr.length, 2); - assertEquals(arr[0], "x"); - assertEquals(arr[1], "y"); + assertEquals(arr, ["x", "y"]); } ); @@ -194,10 +196,9 @@ testWalk( await touch(d + "/y"); }, async function skip(): Promise { - assertReady(2); + assertReady(3); const arr = await walkArray(".", { skip: [/x/] }); - assertEquals(arr.length, 1); - assertEquals(arr[0], "y"); + assertEquals(arr, [".", "y"]); } ); @@ -208,10 +209,9 @@ testWalk( await touch(d + "/z"); }, async function skipAny(): Promise { - assertReady(3); + assertReady(4); const arr = await walkArray(".", { skip: [/x/, /y/] }); - assertEquals(arr.length, 1); - assertEquals(arr[0], "z"); + assertEquals(arr, [".", "z"]); } ); @@ -224,19 +224,18 @@ testWalk( await touch(d + "/b/z"); }, async function subDir(): Promise { - assertReady(3); + assertReady(6); const arr = await walkArray("b"); - assertEquals(arr.length, 1); - assertEquals(arr[0], "b/z"); + assertEquals(arr, ["b", "b/z"]); } ); testWalk( async (_d: string): Promise => {}, async function onError(): Promise { - assertReady(0); + assertReady(1); const ignored = await walkArray("missing"); - assertEquals(ignored.length, 0); + assertEquals(ignored, ["missing"]); let errors = 0; await walkArray("missing", { onError: (_e): number => (errors += 1) }); // It's 2 since walkArray iterates over both sync and async. @@ -265,7 +264,7 @@ testWalk( return; } - assertReady(3); + assertReady(6); const files = await walkArray("a"); assertEquals(files.length, 2); assert(!files.includes("a/bb/z")); diff --git a/prettier/main.ts b/prettier/main.ts index 1afda89d1fbad66702c5e9a24f31bbbf99550315..bd9238be20b87d1f4a771cd4ae35802347825fac 100755 --- a/prettier/main.ts +++ b/prettier/main.ts @@ -23,11 +23,10 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. // This script formats the given source files. If the files are omitted, it // formats the all files in the repository. -const { args, exit, readFile, writeFile, stdout, stdin, readAll } = Deno; -import { glob, isGlob, GlobOptions } from "../fs/glob.ts"; -import { walk, WalkInfo } from "../fs/walk.ts"; import { parse } from "../flags/mod.ts"; +import { ExpandGlobOptions, WalkInfo, expandGlob } from "../fs/mod.ts"; import { prettier, prettierPlugins } from "./prettier.ts"; +const { args, cwd, exit, readAll, readFile, stdin, stdout, writeFile } = Deno; const HELP_MESSAGE = ` Formats the given files. If no arg is passed, then formats the all files. @@ -289,65 +288,47 @@ async function formatFromStdin( /** * Get the files to format. - * @param selectors The glob patterns to select the files. - * eg `cmd/*.ts` to select all the typescript files in cmd + * @param include The glob patterns to select the files. + * eg `"cmd/*.ts"` to select all the typescript files in cmd * directory. - * eg `cmd/run.ts` to select `cmd/run.ts` file as only. - * @param ignore The glob patterns to ignore files. - * eg `*_test.ts` to ignore all the test file. - * @param options options to pass to `glob(selector, options)` + * eg `"cmd/run.ts"` to select `cmd/run.ts` file as only. + * @param exclude The glob patterns to ignore files. + * eg `"*_test.ts"` to ignore all the test file. + * @param root The directory from which to apply default globs. * @returns returns an async iterable object */ -function getTargetFiles( - selectors: string[], - ignore: string[] = [], - options: GlobOptions = {} +async function* getTargetFiles( + include: string[], + exclude: string[], + root: string = cwd() ): AsyncIterableIterator { - const matchers: Array = []; - - const selectorMap: { [k: string]: boolean } = {}; + const expandGlobOpts: ExpandGlobOptions = { + root, + exclude, + includeDirs: true, + extended: true, + globstar: true + }; - for (const selector of selectors) { - // If the selector already exists then ignore it. - if (selectorMap[selector]) continue; - selectorMap[selector] = true; - if (isGlob(selector) || selector === ".") { - matchers.push(glob(selector, options)); - } else { - matchers.push(selector); + async function* expandDirectory(d: string): AsyncIterableIterator { + for await (const walkInfo of expandGlob("**/*", { + ...expandGlobOpts, + root: d, + includeDirs: false + })) { + yield walkInfo; } } - const skip = ignore.map((i: string): RegExp => glob(i, options)); - - return (async function*(): AsyncIterableIterator { - for (const match of matchers) { - if (typeof match === "string") { - const fileInfo = await Deno.stat(match); - - if (fileInfo.isDirectory()) { - const files = walk(match, { skip }); - - for await (const info of files) { - yield info; - } - } else { - const info: WalkInfo = { - filename: match, - info: fileInfo - }; - - yield info; - } + for (const globString of include) { + for await (const walkInfo of expandGlob(globString, expandGlobOpts)) { + if (walkInfo.info.isDirectory()) { + yield* expandDirectory(walkInfo.filename); } else { - const files = walk(".", { match: [match], skip }); - - for await (const info of files) { - yield info; - } + yield walkInfo; } } - })(); + } } async function main(opts): Promise { @@ -371,12 +352,10 @@ async function main(opts): Promise { console.log(HELP_MESSAGE); exit(0); } - const options: GlobOptions = { flags: "g" }; const files = getTargetFiles( args.length ? args : ["."], - Array.isArray(ignore) ? ignore : [ignore], - options + Array.isArray(ignore) ? ignore : [ignore] ); const tty = Deno.isTTY(); diff --git a/testing/runner.ts b/testing/runner.ts index b0357d8e4530bc4262519e900a0d67069dcad209..a78ed2b3af4445281a9a98019b37e4e0a5490cb7 100755 --- a/testing/runner.ts +++ b/testing/runner.ts @@ -1,14 +1,9 @@ #!/usr/bin/env -S deno -A // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import { parse } from "../flags/mod.ts"; -import { - WalkInfo, - expandGlobSync, - glob, - ExpandGlobOptions -} from "../fs/mod.ts"; +import { ExpandGlobOptions, expandGlob } from "../fs/mod.ts"; import { isWindows } from "../fs/path/constants.ts"; -import { isAbsolute, join } from "../fs/path/mod.ts"; +import { join } from "../fs/path/mod.ts"; import { RunTestsOptions, runTests } from "./mod.ts"; const { DenoError, ErrorKind, args, cwd, exit } = Deno; @@ -64,60 +59,56 @@ function filePathToUrl(path: string): string { return `file://${isWindows ? "/" : ""}${path.replace(/\\/g, "/")}`; } -function expandDirectory(dir: string, options: ExpandGlobOptions): WalkInfo[] { - return DIR_GLOBS.flatMap((s: string): WalkInfo[] => [ - ...expandGlobSync(s, { ...options, root: dir }) - ]); -} - /** * Given a list of globs or URLs to include and exclude and a root directory - * from which to expand relative globs, return a list of URLs + * from which to expand relative globs, yield a list of URLs * (file: or remote) that should be imported for the test runner. */ -export async function findTestModules( +export async function* findTestModules( includeModules: string[], excludeModules: string[], root: string = cwd() -): Promise { +): AsyncIterableIterator { const [includePaths, includeUrls] = partition(includeModules, isRemoteUrl); const [excludePaths, excludeUrls] = partition(excludeModules, isRemoteUrl); - const expandGlobOpts = { + const expandGlobOpts: ExpandGlobOptions = { root, + exclude: excludePaths, + includeDirs: true, extended: true, - globstar: true, - filepath: true + globstar: true }; - // TODO: We use the `g` flag here to support path prefixes when specifying - // excludes. Replace with a solution that does this more correctly. - const excludePathPatterns = excludePaths.map( - (s: string): RegExp => - glob(isAbsolute(s) ? s : join(root, s), { ...expandGlobOpts, flags: "g" }) - ); + async function* expandDirectory(d: string): AsyncIterableIterator { + for (const dirGlob of DIR_GLOBS) { + for await (const walkInfo of expandGlob(dirGlob, { + ...expandGlobOpts, + root: d, + includeDirs: false + })) { + yield filePathToUrl(walkInfo.filename); + } + } + } + + for (const globString of includePaths) { + for await (const walkInfo of expandGlob(globString, expandGlobOpts)) { + if (walkInfo.info.isDirectory()) { + yield* expandDirectory(walkInfo.filename); + } else { + yield filePathToUrl(walkInfo.filename); + } + } + } + const excludeUrlPatterns = excludeUrls.map( (url: string): RegExp => RegExp(url) ); - const notExcludedPath = ({ filename }: WalkInfo): boolean => - !excludePathPatterns.some((p: RegExp): boolean => !!filename.match(p)); - const notExcludedUrl = (url: string): boolean => + const shouldIncludeUrl = (url: string): boolean => !excludeUrlPatterns.some((p: RegExp): boolean => !!url.match(p)); - const matchedPaths = includePaths - .flatMap((s: string): WalkInfo[] => [...expandGlobSync(s, expandGlobOpts)]) - .filter(notExcludedPath) - .flatMap(({ filename, info }): string[] => - info.isDirectory() - ? expandDirectory(filename, { ...expandGlobOpts, includeDirs: false }) - .filter(notExcludedPath) - .map(({ filename }): string => filename) - : [filename] - ); - - const matchedUrls = includeUrls.filter(notExcludedUrl); - - return [...matchedPaths.map(filePathToUrl), ...matchedUrls]; + yield* includeUrls.filter(shouldIncludeUrl); } export interface RunTestModulesOptions extends RunTestsOptions { @@ -162,9 +153,13 @@ export async function runTestModules({ skip = /^\s*$/, disableLog = false }: RunTestModulesOptions = {}): Promise { - const testModuleUrls = await findTestModules(include, exclude); + let moduleCount = 0; + for await (const testModule of findTestModules(include, exclude)) { + await import(testModule); + moduleCount++; + } - if (testModuleUrls.length == 0) { + if (moduleCount == 0) { const noneFoundMessage = "No matching test modules found."; if (!allowNone) { throw new DenoError(ErrorKind.NotFound, noneFoundMessage); @@ -175,11 +170,7 @@ export async function runTestModules({ } if (!disableLog) { - console.log(`Found ${testModuleUrls.length} matching test modules.`); - } - - for (const url of testModuleUrls) { - await import(url); + console.log(`Found ${moduleCount} matching test modules.`); } await runTests({ diff --git a/testing/runner_test.ts b/testing/runner_test.ts index 9b6214918b54903cfa088e455e31ea00611dc07a..e2617b1552b77e22d7b5adeec3a403b215020776 100644 --- a/testing/runner_test.ts +++ b/testing/runner_test.ts @@ -3,17 +3,30 @@ import { test } from "./mod.ts"; import { findTestModules } from "./runner.ts"; import { isWindows } from "../fs/path/constants.ts"; import { assertEquals } from "../testing/asserts.ts"; +const { cwd } = Deno; function urlToFilePath(url: URL): string { // Since `new URL('file:///C:/a').pathname` is `/C:/a`, remove leading slash. return url.pathname.slice(url.protocol == "file:" && isWindows ? 1 : 0); } +async function findTestModulesArray( + include: string[], + exclude: string[], + root: string = cwd() +): Promise { + const result = []; + for await (const testModule of findTestModules(include, exclude, root)) { + result.push(testModule); + } + return result; +} + const TEST_DATA_URL = new URL("testdata", import.meta.url); const TEST_DATA_PATH = urlToFilePath(TEST_DATA_URL); test(async function findTestModulesDir1(): Promise { - const urls = await findTestModules(["."], [], TEST_DATA_PATH); + const urls = await findTestModulesArray(["."], [], TEST_DATA_PATH); assertEquals(urls.sort(), [ `${TEST_DATA_URL}/bar_test.js`, `${TEST_DATA_URL}/foo_test.ts`, @@ -27,7 +40,7 @@ test(async function findTestModulesDir1(): Promise { }); test(async function findTestModulesDir2(): Promise { - const urls = await findTestModules(["subdir"], [], TEST_DATA_PATH); + const urls = await findTestModulesArray(["subdir"], [], TEST_DATA_PATH); assertEquals(urls.sort(), [ `${TEST_DATA_URL}/subdir/bar_test.js`, `${TEST_DATA_URL}/subdir/foo_test.ts`, @@ -37,7 +50,11 @@ test(async function findTestModulesDir2(): Promise { }); test(async function findTestModulesGlob(): Promise { - const urls = await findTestModules(["**/*_test.{js,ts}"], [], TEST_DATA_PATH); + const urls = await findTestModulesArray( + ["**/*_test.{js,ts}"], + [], + TEST_DATA_PATH + ); assertEquals(urls.sort(), [ `${TEST_DATA_URL}/bar_test.js`, `${TEST_DATA_URL}/foo_test.ts`, @@ -47,7 +64,7 @@ test(async function findTestModulesGlob(): Promise { }); test(async function findTestModulesExcludeDir(): Promise { - const urls = await findTestModules(["."], ["subdir"], TEST_DATA_PATH); + const urls = await findTestModulesArray(["."], ["subdir"], TEST_DATA_PATH); assertEquals(urls.sort(), [ `${TEST_DATA_URL}/bar_test.js`, `${TEST_DATA_URL}/foo_test.ts`, @@ -57,7 +74,7 @@ test(async function findTestModulesExcludeDir(): Promise { }); test(async function findTestModulesExcludeGlob(): Promise { - const urls = await findTestModules(["."], ["**/foo*"], TEST_DATA_PATH); + const urls = await findTestModulesArray(["."], ["**/foo*"], TEST_DATA_PATH); assertEquals(urls.sort(), [ `${TEST_DATA_URL}/bar_test.js`, `${TEST_DATA_URL}/subdir/bar_test.js`, @@ -73,6 +90,6 @@ test(async function findTestModulesRemote(): Promise { "https://example.com/colors_test.ts", "http://example.com/printf_test.ts" ]; - const matches = await findTestModules(urls, []); + const matches = await findTestModulesArray(urls, []); assertEquals(matches, urls); });