test: add python code for grammer test

trial/pkg/application/pyapp/py_ident_app_test.go

 package pyapp
 
 import (
+	"fmt"
 	. "github.com/onsi/gomega"
+	"github.com/phodal/coca/pkg/adapter/cocafile"
 	"io/ioutil"
+	"strings"
 	"testing"
 )
 
 
+func Test_AllPythonGrammar(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	app := new(PythonApiApp)
+
+	var PyFileFilter = func(path string) bool {
+		return strings.HasSuffix(path, ".py")
+	}
+	files := cocafile.GetFilesWithFilter("testdata/grammar/", PyFileFilter)
+
+	for _, file := range files {
+		fmt.Println(file)
+		file, _ := ioutil.ReadFile(file)
+		app.Analysis(string(file), "")
+	}
+
+	g.Expect(1).To(Equal(1))
+}
+
 func Test_PythonClass(t *testing.T) {
 	g := NewGomegaWithT(t)
 
@@ -18,7 +40,6 @@ func Test_PythonClass(t *testing.T) {
 	g.Expect(1).To(Equal(1))
 }
 
-
 func Test_PythonFuncDefStm(t *testing.T) {
 	g := NewGomegaWithT(t)
 

trial/pkg/application/pyapp/testdata/grammar/argument.py

+# argument
+#     : test (comp_for | ASSIGN test)?
+#     | (POWER | STAR) test
+#     ;
+
+# test
+b(x)
+
+# test comp_for
+b(x for x in a)
+
+# test ASSIGN test
+b(x=i)
+
+# test COMMA test ASSIGN test COMMA test ASSIGN test
+b(z, x=i, y=u)
+
+# POWER test
+b(**z)
+
+# STAR test
+b(*z)
+
+# test COMMA STAR test COMMA test ASSIGN test
+b(y, *z, x=i)

trial/pkg/application/pyapp/testdata/grammar/atom.py

+# atom
+#     : OPEN_PAREN (yield_expr | testlist_comp)? CLOSE_PAREN
+#     | OPEN_BRACKET testlist_comp? CLOSE_BRACKET
+#     | OPEN_BRACE dictorsetmaker? CLOSE_BRACE
+#     | REVERSE_QUOTE testlist COMMA? REVERSE_QUOTE
+#     | dotted_name
+#     | ELLIPSIS
+#     | name
+#     | PRINT
+#     | EXEC
+#     | MINUS? number
+#     | NONE
+#     | STRING+
+#     ;
+
+# OPEN_PAREN CLOSE_PAREN
+()
+
+# OPEN_PAREN yield_expr CLOSE_PAREN
+def f():
+    (yield)
+
+# OPEN_PAREN testlist_comp CLOSE_PAREN
+(x, a, q == 1)
+
+# OPEN_BRACKET CLOSE_BRACKET
+[]
+
+# OPEN_BRACKET testlist_comp CLOSE_BRACKET
+[1, 3, b, p == 1]
+
+# OPEN_BRACE CLOSE_BRACE
+{}
+
+# OPEN_BRACE dictorsetmaker CLOSE_BRACE
+{x : y for x, y in a}
+
+# dotted_name
+b.a
+
+# ELLIPSIS
+...
+
+# name
+f
+
+# number
+90
+
+# MINUS number
+-1
+
+# NONE
+None
+
+# STRING
+"12312313"
+
+# STRING STRING
+"1231231" "123151"

trial/pkg/application/pyapp/testdata/grammar/classdef.py

+# classdef: CLASS name (OPEN_PAREN arglist? CLOSE_PAREN)? COLON suite
+
+# CLASS NAME COLON suite
+class foo: pass
+
+# CLASS NAME OPEN_PAREN CLOSE_PAREN COLON suite
+class bar(): pass
+
+# CLASS NAME OPEN_PAREN arglist CLOSE_PAREN COLON suite
+class baz(foo):
+    pass

trial/pkg/application/pyapp/testdata/grammar/comprehension.py

+# comp_for
+#     : FOR exprlist IN logical_test comp_iter?
+#     ;
+#
+# comp_iter
+#     : comp_for
+#     | IF test comp_iter?
+#     ;
+
+# FOR exprlist IN logical_test
+[x for x in a]
+
+# FOR exprlist IN logical_test comp_for
+[x for x in a for a in k]
+
+# FOR exprlist IN logical_test IF test
+[x for x in a if x == 1]
+
+# FOR exprlist IN logical_test IF test IF test
+[x for x in a if x == 1 if x != 9]
+
+# FOR exprlist IN logical_test IF test comp_for
+[x for x in a if x == 1 for a in q]

trial/pkg/application/pyapp/testdata/grammar/def_parameters.py

+# def_parameters: def_parameter (COMMA def_parameter)*;
+# def_parameter: named_parameter (ASSIGN test)?;
+# named_parameter: NAME (COLON test)?;
+
+# NAME COLON test
+def single_typed(x: int): pass
+
+# NAME COLON test ASSIGN test
+def single_default_typed(x: int = 4): pass
+
+# NAME COMMA NAME ASSIGN test
+def second_default(x, y = 4): pass

trial/pkg/application/pyapp/testdata/grammar/dictorsetmaker.py

+# dictorsetmaker
+#     : (test COLON test | POWER expr) (COMMA (test COLON test | POWER expr))* COMMA? // key_datum_list
+#     | test COLON test comp_for                                                      // dict_comprehension
+#     | testlist_comp
+#     ;
+
+# test COLON test
+{d : y}
+
+# POWER expr
+{**x}
+
+# test COLON test COMMA test COLON test
+{d : y, x : z}
+
+# test COLON test COMMA test COLON test COMMA
+{d : y, x : z,}
+
+# POWER expr COMMA test COLON test
+{**x, d : y}
+
+# test COLON test COMMA POWER expr
+{d : y, **x}
+
+# test COLON test comp_for
+{d : y for d, y in x}
+
+# testlist_comp
+{x, q, y}

trial/pkg/application/pyapp/testdata/grammar/expr.py

+# expr
+#     : AWAIT? atom trailer*
+#     | <assoc=right> expr op=POWER expr
+#     | op=(ADD | MINUS | NOT_OP) expr
+#     | expr op=(STAR | DIV | MOD | IDIV | AT) expr
+#     | expr op=(ADD | MINUS) expr
+#     | expr op=(LEFT_SHIFT | RIGHT_SHIFT) expr
+#     | expr op=AND_OP expr
+#     | expr op=XOR expr
+#     | expr op=OR_OP expr
+#     ;
+
+# atom
+"123"
+
+# AWAIT atom
+await 5
+
+# atom trailer
+"1231".lower()
+
+# atom trailer trailer
+"1231".lower().upper()
+
+# AWAIT atom trailer trailer
+await "1231".lower().upper()
+
+# expr op=POWER expr op=POWER expr
+2 ** 2 ** 3
+
+# ADD expr
++6
+
+# MINUS expr
+-6
+
+# NOT_OP expr
+~6
+
+# expr STAR expr
+6 * 7
+
+# expr DIV expr
+6 / 7
+
+# expr MOD expr
+6 % 8
+
+# expr IDIV expr
+6 // 7
+
+# expr AT expr
+6 @ 2
+
+# expr ADD expr
+6 + 1
+
+# expr MINUS expr
+7 - 9
+
+# expr LEFT_SHIFT expr
+8 << 9
+
+# expr RIGHT_SHIFT expr
+4 >> 1
+
+# expr op=AND_OP expr
+4 & 6
+
+# expr op=XOR expr
+4 ^ 7
+
+# expr op=OR_OP expr
+7 | 1

trial/pkg/application/pyapp/testdata/grammar/expr_stmt.py

+# expr_stmt: testlist_star_expr assign_part?
+#
+# testlist_star_expr : (test | star_expr) (COMMA (test | star_expr))* COMMA?
+#
+# assign_part
+#     : (ASSIGN testlist_star_expr)* (ASSIGN yield_expr)?
+#     | COLON test (ASSIGN testlist)?
+#     | op=( ADD_ASSIGN
+#          | SUB_ASSIGN
+#          | MULT_ASSIGN
+#          | AT_ASSIGN
+#          | DIV_ASSIGN
+#          | MOD_ASSIGN
+#          | AND_ASSIGN
+#          | OR_ASSIGN
+#          | XOR_ASSIGN
+#          | LEFT_SHIFT_ASSIGN
+#          | RIGHT_SHIFT_ASSIGN
+#          | POWER_ASSIGN
+#          | IDIV_ASSIGN
+#          )
+#       (yield_expr | testlist)
+#     ;
+
+# Yield tests (should not be used outside the function)
+def f():
+    # test ASSIGN yield_expr
+    x = yield
+
+    # test COMMA test COMMA ASSIGN test COMMA star_expr ASSIGN yield_expr
+    x, [*t], = y, *z = yield
+
+    # test '+=' yield_expr
+    x += yield
+
+# test ASSIGN testlist_star_expr
+z = 1, *y
+
+# test ASSIGN test COMMA test ASSIGN test COMMA star_expr COMMA
+x = y, a = z, *q,
+
+# test COLON test
+x: int
+
+# COLON test ASSIGN testlist
+x: int = 8
+
+# test '-=' testlist
+x -= 9

trial/pkg/application/pyapp/testdata/grammar/for_stmt.py

+# for_stmt: ASYNC? FOR exprlist IN testlist COLON suite else_clause?
+
+# FOR exprlist IN testlist COLON suite
+for x in range(1):
+    pass
+
+# FOR exprlist IN testlist COLON suite (else_clause)?
+for x in range(1):
+    x
+else:
+    pass
+
+# async_stmt must be inside async function
+async def f():
+    # ASYNC for_stmt
+    async for _ in range(5):
+        pass

trial/pkg/application/pyapp/testdata/grammar/funcdef.py

+# funcdef: ASYNC? DEF name OPEN_PAREN typedargslist? CLOSE_PAREN (ARROW test)? COLON suite
+
+# DEF NAME OPEN_PAREN CLOSE_PAREN COLON suite
+def foo(): pass
+
+# ASYNC DEF NAME OPEN_PAREN typedargslist? CLOSE_PAREN COLON suite
+async def bar(one): pass
+
+# DEF NAME OPEN_PAREN typedargslist? CLOSE_PAREN ARROW test COLON suite
+def baz(one, two) -> int: pass

trial/pkg/application/pyapp/testdata/grammar/if_stmt.py

+# if_stmt: IF cond=test COLON suite elif_clause* else_clause?
+
+# IF test COLON suite
+if x == 5: pass
+
+# IF test COLON suite elif_clause
+if x == 4:
+    pass
+elif x == 3:
+    pass
+
+# IF test COLON suite else_clause
+if x == 7:
+    pass
+else:
+    pass
+
+# IF test COLON suite elif_clause elif_clause else_clause
+if x == 4:
+    pass
+elif x == 3:
+    pass
+else:
+    pass

trial/pkg/application/pyapp/testdata/grammar/import_stmt.py

+
+# First, check this one
+# import_stmt: IMPORT dotted_as_names
+# dotted_as_names: dotted_as_name (COMMA dotted_as_name)*;
+# dotted_as_name: dotted_name (AS NAME)?;
+
+# IMPORT dotted_name
+import collections.abc
+
+# IMPORT dotted_name AS NAME
+import collections.abc as ss
+
+# IMPORT dotted_name AS NAME COMMA dotted_name
+import collections.abc as ss, itertools
+
+# Then check this
+# import_from: FROM   ((DOT | ELLIPSIS)* dotted_name | (DOT | ELLIPSIS)+)
+#              IMPORT (STAR | OPEN_PAREN import_as_names CLOSE_PAREN | import_as_names);
+# import_as_names: import_as_name (COMMA import_as_name)*;
+# import_as_name: NAME (AS NAME)?;
+
+# FROM dotted_name IMPORT import_as_name
+from classdef import bar as b
+
+# FROM DOT IMPORT import_as_name
+from . import bar
+
+# FROM DOT DOT DOT IMPORT import_as_name
+from . . . import bar
+
+# FROM DOT dotted_name IMPORT import_as_name
+from .classdef import bar
+
+# FROM dotted_name IMPORT STAR
+from sys import *
+
+# FROM dotted_name IMPORT import_as_name COMMA import_as_name
+from collections import bar, baz
+
+# FROM dotted_name IMPORT OPEN_PAREN import_as_name CLOSE_PAREN
+from collections import (bar)

trial/pkg/application/pyapp/testdata/grammar/logical_test.py

+# logical_test
+#     : comparison
+#     | NOT logical_test
+#     | logical_test op=AND logical_test
+#     | logical_test op=OR logical_test
+#     ;
+#
+# comparison
+#     : comparison (LESS_THAN | GREATER_THAN | EQUALS | GT_EQ | LT_EQ | NOT_EQ_1 | NOT_EQ_2 | optional=NOT? IN | IS optional=NOT?) comparison
+#     | expr
+#     ;
+
+# expr EQUALS expr
+a == b
+
+# not expr
+not a
+
+# expr AND expr
+a and b
+
+# expr OR expr
+a or b
+
+# expr LESS_THAN expr
+a < b
+
+# expr GREATER_THAN expr
+a > b
+
+# expr GT_EQ expr
+a >= b
+
+# expr LT_EQ expr
+a <= b
+
+# expr NOT_EQ_2 expr
+a != b
+
+# expr IN expr
+a in b
+
+# expr NOT IN expr
+a not in b
+
+# expr IS expr
+a is b
+
+# expr IS NOT expr
+a is not b

trial/pkg/application/pyapp/testdata/grammar/simple_stmt.py

+# simple_stmt: small_stmt (SEMI_COLON small_stmt)* SEMI_COLON? (NEWLINE | EOF)
+
+# small_stmt NEWLINE
+x = 5
+
+# small_stmt SEMI_COLON small_stmt NEWLINE
+x = 5 ; y = 7
+
+# small_stmt SEMI_COLON NEWLINE
+x = 5 ;
+
+# small_stmt SEMI_COLON small_stmt SEMI_COLON small_stmt SEMI_COLON small_stmt SEMI_COLON EOF
+x = 5 ; y = 7 ; z = 9 ;
\ No newline at end of file

trial/pkg/application/pyapp/testdata/grammar/small_stmt.py

+# del_stmt: DEL exprlist
+
+del x, d[1], await f
+
+# pass_stmt: PASS
+for i in u: pass
+
+# break_stmt: BREAK
+for i in u: break
+
+# continue_stmt: CONTINUE
+for i in u: continue
+
+# return_stmt: RETURN testlist?
+def f():
+    # RETURN
+    return
+def g():
+    # RETURN testlist
+    return 1, 3
+
+# [Python 3] raise_stmt: RAISE (test (FROM test)?)?
+
+# RAISE test FROM test
+raise a from b
+
+# RAISE test
+raise a
+
+try:
+    a
+except:
+    # RAISE
+    raise
+
+# global_stmt: GLOBAL name (COMMA name)*
+
+# GLOBAL name
+global a
+
+# GLOBAL name COMMA name
+global a, b
+
+# assert_stmt: ASSERT test (COMMA test)?
+
+# ASSERT test
+assert a
+
+# ASSERT test COMMA test
+assert a, b
+
+# nonlocal_stmt: NONLOCAL name (COMMA name)*
+
+# NONLOCAL name
+nonlocal a
+
+# NONLOCAL name (COMMA name)
+nonlocal a, v

trial/pkg/application/pyapp/testdata/grammar/subscript.py

+# subscript
+#     : ELLIPSIS
+#     | test
+#     | test? COLON test? sliceop?
+#     ;
+#
+# sliceop
+#     : COLON test?
+#     ;
+
+# ELLIPSIS
+b[...]
+
+# test
+b[a]
+
+# COLON
+b[:]
+
+# test COLON
+b[a:]
+
+# COLON test
+b[:a]
+
+# test COLON test
+b[a:a]
+
+# COLON COLON
+b[::]
+
+# COLON COLON test
+b[::-1]
+
+# test COLON COLON test
+b[a::2]
+
+# COLON test COLON test
+b[:a:2]
+
+# test COLON test COLON test
+b[1:a:2]

trial/pkg/application/pyapp/testdata/grammar/test.py

+# test
+#     : logical_test (IF logical_test ELSE test)?
+#     | LAMBDA varargslist? COLON test
+#     ;
+
+# logical_test
+x == y
+
+# logical_test IF logical_test ELSE test
+x == y if z == b else a == u
+
+# LAMBDA COLON test
+lambda: a
+
+# LAMBDA varargslist COLON test
+lambda x, y: a

trial/pkg/application/pyapp/testdata/grammar/testlist_comp.py

+# testlist_comp
+#     : (test | star_expr) (comp_for | (COMMA (test | star_expr))* COMMA?)
+#     ;
+
+# test
+[x]
+
+# star_expr comp_for
+[z for z in a]
+
+# test COMMA star_expr COMMA
+[x, *a,]
+
+# star_expr COMMA test COMMA star_expr
+[*u, a, *i]

trial/pkg/application/pyapp/testdata/grammar/tokens.py

+DECIMAL = 0000000000
+DECIMAL = 1234567890
+OCT_1   = 0o01234567
+OCT_2   = 0O01234567
+HEX_1   = 0x0123456789abcdef
+HEX_2   = 0X0123456789ABCDEF
+BIN_1   = 0b01
+BIN_1   = 0B01
+IMAG_1  = 0123456789.0123456789j
+IMAG_2  = 0123456789J
+FLOAT_1 = 0123456789.e1234567890
+FLOAT_2 = .0123456789E1234567890
+
+LINE_JOIN_EXPR  = 2 + \
+    2
+
+SHORT_STRING_1 = 'a \'\\ b'
+SHORT_STRING_2 = "a \"\\ b"
+LONG_STRING_1  = b""" asdf " qwer 
+zxcv
+"""
+LONG_STRING_1  = r''' aasdf ' qwer 
+zxcv
+'''
+STRING_WITH_LINE_JOIND = "a \
+b"
+
+# COMMENT
+

trial/pkg/application/pyapp/testdata/grammar/trailer.py

+# trailer
+#     : OPEN_PAREN (argument (COMMA argument)* COMMA?)? CLOSE_PAREN
+#     | OPEN_BRACKET subscript (COMMA subscript)* COMMA? CLOSE_BRACKET
+#     | DOT name
+#     ;
+
+# DOT name
+a.b
+
+# OPEN_PAREN CLOSE_PAREN
+x()
+
+# OPEN_PAREN argument CLOSE_PAREN
+x(a)
+
+# OPEN_PAREN argument COMMA argument COMMA CLOSE_PAREN
+x(a, b,)
+
+# OPEN_PAREN argument COMMA argument COMMA argument CLOSE_PAREN
+x(a, b, c)
+
+# OPEN_BRACKET subscript CLOSE_BRACKET
+x[a]
+
+# OPEN_BRACKET subscript COMMA subscript COMMA CLOSE_BRACKET
+x[a, b,]
+
+# OPEN_BRACKET subscript COMMA subscript COMMA subscript CLOSE_BRACKET
+x[a, b, c]
\ No newline at end of file

trial/pkg/application/pyapp/testdata/grammar/try_stmt.py

+# try_stmt: TRY COLON suite (except_clause+ else_clause? finaly_clause? | finaly_clause)
+
+# TRY COLON suite except_clause
+try:
+    pass
+except:
+    pass
+
+# TRY COLON suite except_clause except_clause else_clause
+try:
+    pass
+except Exception as ex:
+    pass
+except:
+    pass
+else:
+    pass
+
+# TRY COLON suite except_clause finaly_clause
+try:
+    pass
+except Exception:
+    pass
+finally:
+    pass
+
+# TRY COLON suite finaly_clause
+try:
+    pass
+finally:
+    pass

trial/pkg/application/pyapp/testdata/grammar/typedargslist.py

+# typedargslist
+#     : (def_parameters COMMA)? (args (COMMA def_parameters)? (COMMA kwargs)? | kwargs)
+#     | def_parameters
+#     ;
+
+# def_parameters COMMA
+def single(x, *, i): pass
+
+# def_parameters COMMA kwargs
+def f1(x, y, **z): pass
+
+# def_parameters COMMA args COMMA def_parameters COMMA kwargs COMMA
+def f1(x, y, *z: int, a, b, **c: int,): pass
+
+# def_parameters COMMA args
+def f1(x, y, *z): pass
+

trial/pkg/application/pyapp/testdata/grammar/varargslist.py

+# varargslist
+#     : (vardef_parameters COMMA)? (varargs (COMMA vardef_parameters)? (COMMA varkwargs)? | varkwargs) COMMA?
+#     | vardef_parameters COMMA?
+#     ;
+#
+# vardef_parameters
+#     : vardef_parameter (COMMA vardef_parameter)*
+#     ;
+#
+# vardef_parameter
+#     : name (ASSIGN test)?
+#     ;
+#
+# varargs
+#     : STAR name
+#     ;
+#
+# varkwargs
+#     : POWER name
+#     ;
+
+# vardef_parameters COMMA
+# NAME COMMA
+lambda x,: 5
+
+# vardef_parameters COMMA
+# NAME COMMA STAR COMMA NAME COMMA
+lambda x, *, y,: 5
+
+# vardef_parameters
+# NAME COMMA STAR COMMA NAME ASSIGN test
+lambda x, *, y=7: 5
+
+# varargs
+lambda *y: 8
+
+# varargs COMMA vardef_parameters COMMA
+lambda *y, z: 8
+
+# vardef_parameters COMMA varargs COMMA vardef_parameters COMMA
+lambda a, b, *y, z: 8
+
+# vardef_parameters COMMA varargs COMMA vardef_parameters COMMA varkwargs
+lambda a, b, *y, z, **k: 8
+
+# varkwargs
+lambda **z: 8
+
+# vardef_parameters COMMA varkwargs
+lambda a, b, c, **k: 8

trial/pkg/application/pyapp/testdata/grammar/while_stmt.py

+# while_stmt: WHILE test COLON suite else_clause?
+
+# WHILE test COLON suite
+while x == 3:
+    x += 1
+
+# WHILE test COLON suite else_clause
+while x == 3:
+    x += 1
+else:
+    pass

trial/pkg/application/pyapp/testdata/grammar/with_stmt.py

+# with_stmt: ASYNC? WITH with_item (COMMA with_item)* COLON suite
+
+# WITH with_item COLON suite
+with open("with_stmt.py"):
+    pass
+
+# WITH with_item COMMA with_item COLON suite
+with open("with_stmt.py") as a, open("with_stmt.py") as b:
+    pass
+
+# async_stmt must be inside async function
+async def f():
+    # ASYNC with_stmt
+    async with open("with_stmt.py") as f:
+        pass

trial/pkg/application/pyapp/testdata/grammar/yield_expr.py

+# yield_expr
+#     : YIELD yield_arg?
+#     ;
+#
+# yield_arg
+#     : FROM test
+#     | testlist
+#     ;
+
+def f():
+
+    # YIELD
+    yield
+
+    # YIELD FROM test
+    yield from g
+
+    # YIELD testlist
+    yield x, a, b if x else a
+