Improve integration test harness (#1142)

tests/001_hello.test

+args: tests/001_hello.js --reload
+output: tests/002_hello.ts.out

tests/002_hello.test

+args: tests/002_hello.ts --reload
+output: tests/002_hello.ts.out

tests/003_relative_import.test

+args: tests/003_relative_import.ts --reload
+output: tests/003_relative_import.ts.out

tests/004_set_timeout.test

+args: tests/004_set_timeout.ts --reload
+output: tests/004_set_timeout.ts.out

tests/005_more_imports.test

+args: tests/005_more_imports.ts --reload
+output: tests/005_more_imports.ts.out

tests/006_url_imports.test

+args: tests/006_url_imports.ts --reload
+output: tests/006_url_imports.ts.out

tests/010_set_interval.test

+args: tests/010_set_interval.ts --reload
+output: tests/010_set_interval.ts.out

tests/012_async.test

+args: tests/012_async.ts --reload
+output: tests/012_async.ts.out

tests/013_dynamic_import.test

+args: tests/013_dynamic_import.ts --reload
+output: tests/013_dynamic_import.ts.out

tests/014_duplicate_import.test

+args: tests/014_duplicate_import.ts --reload
+output: tests/014_duplicate_import.ts.out

tests/015_import_no_ext.test

+args: tests/015_import_no_ext.ts --reload
+output: tests/015_import_no_ext.ts.out

tests/016_double_await.test

+args: tests/016_double_await.ts --reload
+output: tests/016_double_await.ts.out

tests/017_import_redirect.test

+args: tests/017_import_redirect.ts --reload
+output: tests/017_import_redirect.ts.out

tests/018_async_catch.test

+args: tests/018_async_catch.ts --reload
+output: tests/018_async_catch.ts.out

tests/019_media_types.test

+args: tests/019_media_types.ts --reload
+output: tests/019_media_types.ts.out

tests/020_json_modules.test

+args: tests/020_json_modules.ts --reload
+output: tests/020_json_modules.ts.out

tests/README.md

+# Integration Tests
+
+This path contains integration tests. When the integration tests are run, the
+test harness will execute tests which are defined in a `.test` file and located
+in the base of this path.
+
+A `.test` file is a simple configuration format where each option is specified
+on a single line. The key is the string to the left of the `:` deliminator and
+the value is the string to the right.
+
+| Key         | Required | Description                                                                                                                                                               |
+| ----------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `args`      | Yes      | Specifies the command line arguments for the test. This should typically be input script for the test and a `--reload` to help ensure Deno doesn't leverage the cache.    |
+| `output`    | Yes      | This is a text file which represents the output of the command. The string `[WILDCARD]` can be used in the output to specify ranges of text which any output is accepted. |
+| `exit_code` | No       | If not present, it is assumed the script would exit normally (`0`). If specified, the harness will ensure the proper code is received.                                    |

tests/async_error.test

+exit_code: 1
+args: tests/async_error.ts --reload
+output: tests/async_error.ts.out

tests/error_001.test

+args: tests/error_001.ts --reload
+exit_code: 1
+output: tests/error_001.ts.out

tests/error_002.test

+args: tests/error_002.ts --reload
+exit_code: 1
+output: tests/error_002.ts.out

tests/error_003_typescript.test

+args: tests/error_003_typescript.ts --reload
+exit_code: 1
+output: tests/error_003_typescript.ts.out

tests/error_004_missing_module.test

+args: tests/error_004_missing_module.ts --reload
+exit_code: 1
+output: tests/error_004_missing_module.ts.out

tests/error_005_missing_dynamic_import.test

+args: tests/error_005_missing_dynamic_import.ts --reload
+exit_code: 1
+output: tests/error_005_missing_dynamic_import.ts.out

tests/error_006_import_ext_failure.test

+args: tests/error_006_import_ext_failure.ts --reload
+exit_code: 1
+output: tests/error_006_import_ext_failure.ts.out

tests/error_007_any.test

+args: tests/error_007_any.ts --reload
+exit_code: 1
+output: tests/error_007_any.ts.out

tests/error_008_checkjs.test

+args: tests/error_008_checkjs.js --reload
+exit_code: 1
+output: tests/error_008_checkjs.js.out

tests/exit_error42.test

+exit_code: 42
+args: tests/exit_error42.ts --reload
+output: tests/exit_error42.ts.out

tests/https_import.test

+args: tests/https_import.ts --reload
+output: tests/https_import.ts.out

tests/types.out

+// Copyright 2018 the Deno authors. All rights reserved. MIT license.
+
+/// <reference no-default-lib="true" />
+/// <reference lib="esnext" />
+[WILDCARD]
+declare module "deno" {
+[WILDCARD]
+}
+
+declare interface Window {
+[WILDCARD]
+}
+
+declare const window: Window;
+[WILDCARD]

tests/types.test

+args: --types
+output: tests/types.out

tools/check_output_test.py → tools/integration_tests.py

 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 # Copyright 2018 the Deno authors. All rights reserved. MIT license.
-# Given a deno executable, this script execute several integration tests
-# with it. The tests are stored in //tests/ and each script has a corresponding
-# .out file which specifies what the stdout should be.
+# Given a deno executable, this script executes several integration tests with
+# it. The tests are stored in /tests/ and each is specified in a .yaml file
+# where a description, command line, and output are specified.  Optionally an
+# exit code can be specified.
 #
-# Usage: check_output_test.py [path to deno executable]
+# Usage: integration_tests.py [path to deno executable]
 import os
+import re
 import sys
 import subprocess
-from util import pattern_match, parse_exit_code
+from util import pattern_match, green_ok, red_failed
 
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 tests_path = os.path.join(root_path, "tests")
 
 
-def check_output_test(deno_exe_filename):
-    assert os.path.isfile(deno_exe_filename)
-    outs = sorted([
+def read_test(file_name):
+    with open(file_name, "r") as f:
+        test_file = f.read()
+    lines = test_file.splitlines()
+    test_dict = {}
+    for line in lines:
+        key, value = re.split(r":\s+", line)
+        test_dict[key] = value
+    return test_dict
+
+
+def integration_tests(deno_executable):
+    assert os.path.isfile(deno_executable)
+    tests = sorted([
         filename for filename in os.listdir(tests_path)
-        if filename.endswith(".out")
+        if filename.endswith(".test")
     ])
-    assert len(outs) > 1
-    tests = [(os.path.splitext(filename)[0], filename) for filename in outs]
-    for (script, out_filename) in tests:
-        script_abs = os.path.join(tests_path, script)
-        out_abs = os.path.join(tests_path, out_filename)
-        with open(out_abs, 'r') as f:
+    assert len(tests) > 0
+    for test_filename in tests:
+        test_abs = os.path.join(tests_path, test_filename)
+        test = read_test(test_abs)
+        exit_code = int(test.get("exit_code", 0))
+        args = test.get("args", "").split(" ")
+        output_abs = os.path.join(root_path, test.get("output", ""))
+        with open(output_abs, 'r') as f:
             expected_out = f.read()
-        cmd = [deno_exe_filename, script_abs, "--reload"]
-        expected_code = parse_exit_code(script)
+        cmd = [deno_executable] + args
+        print "test %s" % (test_filename)
         print " ".join(cmd)
         actual_code = 0
         try:
@@ -36,28 +52,27 @@ def check_output_test(deno_exe_filename):
         except subprocess.CalledProcessError as e:
             actual_code = e.returncode
             actual_out = e.output
-            if expected_code == 0:
-                print "Expected success but got error. Output:"
-                print actual_out
-                sys.exit(1)
-
-        if expected_code != actual_code:
-            print "Expected exit code %d but got %d" % (expected_code,
-                                                        actual_code)
+
+        if exit_code != actual_code:
+            print "... " + red_failed()
+            print "Expected exit code %d but got %d" % (exit_code, actual_code)
             print "Output:"
             print actual_out
             sys.exit(1)
 
         if pattern_match(expected_out, actual_out) != True:
+            print "... " + red_failed()
             print "Expected output does not match actual."
-            print "Expected: " + expected_out
-            print "Actual:   " + actual_out
+            print "Expected output: \n" + expected_out
+            print "Actual output:   \n" + actual_out
             sys.exit(1)
 
+        print "... " + green_ok()
+
 
 def main(argv):
-    check_output_test(argv[1])
+    integration_tests(argv[1])
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     sys.exit(main(sys.argv))

tools/test.py

@@ -4,7 +4,7 @@
 # Usage: ./tools/test.py out/Debug
 import os
 import sys
-from check_output_test import check_output_test
+from integration_tests import integration_tests
 from deno_dir_test import deno_dir_test
 from setup_test import setup_test
 from util import build_path, enable_ansi_colors, executable_suffix, run, rmtree
@@ -58,7 +58,7 @@ def main(argv):
 
     unit_tests(deno_exe)
 
-    check_output_test(deno_exe)
+    integration_tests(deno_exe)
 
     # TODO We currently skip testing the prompt in Windows completely.
     # Windows does not support the pty module used for testing the permission

tools/util.py

@@ -6,6 +6,10 @@ import stat
 import sys
 import subprocess
 
+RESET = "\x1b[0m"
+FG_RED = "\x1b[31m"
+FG_GREEN = "\x1b[32m"
+
 executable_suffix = ".exe" if os.name == "nt" else ""
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
@@ -73,6 +77,14 @@ def shell_quote(arg):
         return quote(arg)
 
 
+def red_failed():
+    return "%sFAILED%s" % (FG_RED, RESET)
+
+
+def green_ok():
+    return "%sok%s" % (FG_GREEN, RESET)
+
+
 def remove_and_symlink(target, name, target_is_dir=False):
     try:
         # On Windows, directory symlink can only be removed with rmdir().