fixes #7688, fixes #2936, fixes #2867, fixes #2208

extensions/python/OSSREADME.json

 // ATTENTION - THIS DIRECTORY CONTAINS THIRD PARTY OPEN SOURCE MATERIALS:
-[{
-	"name": "textmate/python.tmbundle",
+[
+	{
+		"name": "MagicStack/MagicPython",
 		"version": "0.0.0",
-	"license": "TextMate Bundle License",
-	"repositoryURL": "https://github.com/textmate/python.tmbundle",
-	"licenseDetail": [
-		"Copyright (c) textmate-python.tmbundle project authors",
-		"",
-		"If not otherwise specified (see below), files in this repository fall under the following license:",
-		"",
-		"Permission to copy, use, modify, sell and distribute this",
-		"software is granted. This software is provided \"as is\" without",
-		"express or implied warranty, and with no claim as to its",
-		"suitability for any purpose.",
-		"",
-		"An exception is made for files in readable text which contain their own license information,",
-		"or files where an accompanying file exists (in the same directory) with a \"-license\" suffix added",
-		"to the base-name name of the original file, and an extension of txt, html, or similar. For example",
-		"\"tidy\" is accompanied by \"tidy-license.txt\"."
-	]
-}]
+		"license": "MIT",
+		"repositoryURL": "https://github.com/MagicStack/MagicPython"
+	}
+]
\ No newline at end of file

extensions/python/package.json

@@ -16,14 +16,15 @@
 		"grammars": [{
 			"language": "python",
 			"scopeName": "source.python",
-			"path": "./syntaxes/Python.tmLanguage"
+			"path": "./syntaxes/MagicPython.tmLanguage.json"
 		},{
 			"scopeName": "source.regexp.python",
-			"path": "./syntaxes/Regular Expressions (Python).tmLanguage"
+			"path": "./syntaxes/MagicRegExp.tmLanguage.json"
 		}]
 	},
 	"scripts": {
 		"compile": "gulp compile-extension:python",
-		"watch": "gulp watch-extension:python"
+		"watch": "gulp watch-extension:python",
+		"update-grammar": "node ../../build/npm/update-grammar.js MagicStack/MagicPython grammars/MagicPython.tmLanguage ./syntaxes/MagicPython.tmLanguage.json grammars/MagicRegExp.tmLanguage ./syntaxes/MagicRegExp.tmLanguage.json"
 	}
 }

extensions/python/syntaxes/MagicRegExp.tmLanguage.json

+{
+	"name": "MagicRegExp",
+	"scopeName": "source.regexp.python",
+	"fileTypes": [
+		"re"
+	],
+	"uuid": "39e15186-71e6-11e5-b82c-7c6d62900c7c",
+	"patterns": [
+		{
+			"include": "#regexp-expression"
+		}
+	],
+	"repository": {
+		"regexp-base-expression": {
+			"patterns": [
+				{
+					"name": "support.other.match.any.regexp",
+					"match": "\\."
+				},
+				{
+					"name": "support.other.match.begin.regexp",
+					"match": "\\^"
+				},
+				{
+					"name": "support.other.match.end.regexp",
+					"match": "\\$"
+				},
+				{
+					"name": "keyword.operator.quantifier.regexp",
+					"match": "[+*?]\\??"
+				},
+				{
+					"name": "keyword.operator.disjunction.regexp",
+					"match": "\\|"
+				},
+				{
+					"name": "keyword.operator.quantifier.regexp",
+					"match": "(?x)\n  \\{(\n    \\d+ | \\d+,(\\d+)? | ,\\d+\n  )\\}\n"
+				},
+				{
+					"include": "#regexp-escape-sequence"
+				}
+			]
+		},
+		"regexp-backreference-number": {
+			"name": "meta.backreference.regexp",
+			"match": "(\\\\[1-9]\\d?)",
+			"captures": {
+				"1": {
+					"name": "entity.name.tag.backreference.regexp"
+				}
+			}
+		},
+		"regexp-backreference": {
+			"name": "meta.backreference.named.regexp",
+			"match": "(?x)\n  (\\()  (\\?P= \\w+(?:\\s+[[:alnum:]]+)?)  (\\))\n",
+			"captures": {
+				"1": {
+					"name": "punctuation.parenthesis.backreference.named.begin.regexp support.other.parenthesis.regexp"
+				},
+				"2": {
+					"name": "entity.name.tag.named.backreference.regexp"
+				},
+				"3": {
+					"name": "punctuation.parenthesis.backreference.named.end.regexp support.other.parenthesis.regexp"
+				}
+			}
+		},
+		"regexp-flags": {
+			"name": "storage.modifier.flag.regexp",
+			"match": "\\(\\?[aiLmsux]+\\)"
+		},
+		"regexp-escape-special": {
+			"name": "support.other.escape.special.regexp",
+			"match": "\\\\([AbBdDsSwWZ])"
+		},
+		"regexp-escape-character": {
+			"name": "constant.character.escape.regexp",
+			"match": "(?x)\n  \\\\ (\n        x[0-9A-Fa-f]{2}\n        | 0[0-7]{1,2}\n        | [0-7]{3}\n     )\n"
+		},
+		"regexp-escape-unicode": {
+			"name": "constant.character.unicode.regexp",
+			"match": "(?x)\n  \\\\ (\n        u[0-9A-Fa-f]{4}\n        | U[0-9A-Fa-f]{8}\n     )\n"
+		},
+		"regexp-escape-catchall": {
+			"name": "constant.character.escape.regexp",
+			"match": "\\\\(.|\\n)"
+		},
+		"regexp-escape-sequence": {
+			"patterns": [
+				{
+					"include": "#regexp-escape-special"
+				},
+				{
+					"include": "#regexp-escape-character"
+				},
+				{
+					"include": "#regexp-escape-unicode"
+				},
+				{
+					"include": "#regexp-backreference-number"
+				},
+				{
+					"include": "#regexp-escape-catchall"
+				}
+			]
+		},
+		"regexp-charecter-set-escapes": {
+			"patterns": [
+				{
+					"name": "constant.character.escape.regexp",
+					"match": "\\\\[abfnrtv\\\\]"
+				},
+				{
+					"include": "#regexp-escape-special"
+				},
+				{
+					"name": "constant.character.escape.regexp",
+					"match": "\\\\([0-7]{1,3})"
+				},
+				{
+					"include": "#regexp-escape-character"
+				},
+				{
+					"include": "#regexp-escape-unicode"
+				},
+				{
+					"include": "#regexp-escape-catchall"
+				}
+			]
+		},
+		"regexp-expression": {
+			"patterns": [
+				{
+					"include": "#regexp-base-expression"
+				},
+				{
+					"include": "#regexp-character-set"
+				},
+				{
+					"include": "#regexp-comments"
+				},
+				{
+					"include": "#regexp-flags"
+				},
+				{
+					"include": "#regexp-named-group"
+				},
+				{
+					"include": "#regexp-backreference"
+				},
+				{
+					"include": "#regexp-lookahead"
+				},
+				{
+					"include": "#regexp-lookahead-negative"
+				},
+				{
+					"include": "#regexp-lookbehind"
+				},
+				{
+					"include": "#regexp-lookbehind-negative"
+				},
+				{
+					"include": "#regexp-conditional"
+				},
+				{
+					"include": "#regexp-parentheses-non-capturing"
+				},
+				{
+					"include": "#regexp-parentheses"
+				}
+			]
+		},
+		"regexp-character-set": {
+			"patterns": [
+				{
+					"match": "(?x)\n  \\[ \\^? \\] (?! .*?\\])\n"
+				},
+				{
+					"name": "meta.character.set.regexp",
+					"begin": "(\\[)(\\^)?(\\])?",
+					"end": "(\\])",
+					"beginCaptures": {
+						"1": {
+							"name": "constant.other.set.regexp punctuation.character.set.begin.regexp"
+						},
+						"2": {
+							"name": "keyword.operator.negation.regexp"
+						},
+						"3": {
+							"name": "constant.character.set.regexp"
+						}
+					},
+					"endCaptures": {
+						"1": {
+							"name": "constant.other.set.regexp punctuation.character.set.end.regexp"
+						},
+						"2": {
+							"name": "invalid.illegal.newline.python"
+						}
+					},
+					"patterns": [
+						{
+							"include": "#regexp-charecter-set-escapes"
+						},
+						{
+							"name": "constant.character.set.regexp",
+							"match": "[^\\n]"
+						}
+					]
+				}
+			]
+		},
+		"regexp-named-group": {
+			"name": "meta.named.regexp",
+			"begin": "(?x)\n  (\\()  (\\?P <\\w+(?:\\s+[[:alnum:]]+)?>)\n",
+			"end": "(\\))",
+			"beginCaptures": {
+				"1": {
+					"name": "punctuation.parenthesis.named.begin.regexp support.other.parenthesis.regexp"
+				},
+				"2": {
+					"name": "entity.name.tag.named.group.regexp"
+				}
+			},
+			"endCaptures": {
+				"1": {
+					"name": "punctuation.parenthesis.named.end.regexp support.other.parenthesis.regexp"
+				},
+				"2": {
+					"name": "invalid.illegal.newline.python"
+				}
+			},
+			"patterns": [
+				{
+					"include": "#regexp-expression"
+				}
+			]
+		},
+		"regexp-comments": {
+			"name": "comment.regexp",
+			"begin": "\\(\\?#",
+			"end": "(\\))",
+			"beginCaptures": {
+				"0": {
+					"name": "punctuation.comment.begin.regexp"
+				}
+			},
+			"endCaptures": {
+				"1": {
+					"name": "punctuation.comment.end.regexp"
+				},
+				"2": {
+					"name": "invalid.illegal.newline.python"
+				}
+			},
+			"patterns": [
+				{
+					"include": "#codetags"
+				}
+			]
+		},
+		"regexp-lookahead": {
+			"begin": "(\\()\\?=",
+			"end": "(\\))",
+			"beginCaptures": {
+				"0": {
+					"name": "keyword.operator.lookahead.regexp"
+				},
+				"1": {
+					"name": "punctuation.parenthesis.lookahead.begin.regexp"
+				}
+			},
+			"endCaptures": {
+				"1": {
+					"name": "punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.regexp"
+				},
+				"2": {
+					"name": "invalid.illegal.newline.python"
+				}
+			},
+			"patterns": [
+				{
+					"include": "#regexp-expression"
+				}
+			]
+		},
+		"regexp-lookahead-negative": {
+			"begin": "(\\()\\?!",
+			"end": "(\\))",
+			"beginCaptures": {
+				"0": {
+					"name": "keyword.operator.lookahead.negative.regexp"
+				},
+				"1": {
+					"name": "punctuation.parenthesis.lookahead.begin.regexp"
+				}
+			},
+			"endCaptures": {
+				"1": {
+					"name": "punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.negative.regexp"
+				},
+				"2": {
+					"name": "invalid.illegal.newline.python"
+				}
+			},
+			"patterns": [
+				{
+					"include": "#regexp-expression"
+				}
+			]
+		},
+		"regexp-lookbehind": {
+			"begin": "(\\()\\?<=",
+			"end": "(\\))",
+			"beginCaptures": {
+				"0": {
+					"name": "keyword.operator.lookbehind.regexp"
+				},
+				"1": {
+					"name": "punctuation.parenthesis.lookbehind.begin.regexp"
+				}
+			},
+			"endCaptures": {
+				"1": {
+					"name": "punctuation.parenthesis.lookbehind.end.regexp keyword.operator.lookbehind.regexp"
+				},
+				"2": {
+					"name": "invalid.illegal.newline.python"
+				}
+			},
+			"patterns": [
+				{
+					"include": "#regexp-expression"
+				}
+			]
+		},
+		"regexp-lookbehind-negative": {
+			"begin": "(\\()\\?<!",
+			"end": "(\\))",
+			"beginCaptures": {
+				"0": {
+					"name": "keyword.operator.lookbehind.negative.regexp"
+				},
+				"1": {
+					"name": "punctuation.parenthesis.lookbehind.begin.regexp"
+				}
+			},
+			"endCaptures": {
+				"1": {
+					"name": "punctuation.parenthesis.lookbehind.end.regexp keyword.operator.lookbehind.negative.regexp"
+				},
+				"2": {
+					"name": "invalid.illegal.newline.python"
+				}
+			},
+			"patterns": [
+				{
+					"include": "#regexp-expression"
+				}
+			]
+		},
+		"regexp-conditional": {
+			"begin": "(\\()\\?\\((\\w+(?:\\s+[[:alnum:]]+)?|\\d+)\\)",
+			"end": "(\\))",
+			"beginCaptures": {
+				"0": {
+					"name": "keyword.operator.conditional.regexp"
+				},
+				"1": {
+					"name": "punctuation.parenthesis.conditional.begin.regexp"
+				}
+			},
+			"endCaptures": {
+				"1": {
+					"name": "punctuation.parenthesis.conditional.end.regexp keyword.operator.conditional.negative.regexp"
+				},
+				"2": {
+					"name": "invalid.illegal.newline.python"
+				}
+			},
+			"patterns": [
+				{
+					"include": "#regexp-expression"
+				}
+			]
+		},
+		"regexp-parentheses-non-capturing": {
+			"begin": "\\(\\?:",
+			"end": "(\\))",
+			"beginCaptures": {
+				"0": {
+					"name": "punctuation.parenthesis.non-capturing.begin.regexp support.other.parenthesis.regexp"
+				}
+			},
+			"endCaptures": {
+				"1": {
+					"name": "punctuation.parenthesis.non-capturing.end.regexp support.other.parenthesis.regexp"
+				},
+				"2": {
+					"name": "invalid.illegal.newline.python"
+				}
+			},
+			"patterns": [
+				{
+					"include": "#regexp-expression"
+				}
+			]
+		},
+		"regexp-parentheses": {
+			"begin": "\\(",
+			"end": "(\\))",
+			"beginCaptures": {
+				"0": {
+					"name": "punctuation.parenthesis.begin.regexp support.other.parenthesis.regexp"
+				}
+			},
+			"endCaptures": {
+				"1": {
+					"name": "punctuation.parenthesis.end.regexp support.other.parenthesis.regexp"
+				},
+				"2": {
+					"name": "invalid.illegal.newline.python"
+				}
+			},
+			"patterns": [
+				{
+					"include": "#regexp-expression"
+				}
+			]
+		}
+	},
+	"version": "https://github.com/MagicStack/MagicPython/commit/a45287d159f82256c768ad2e1114ca9751d28670"
+}
\ No newline at end of file

extensions/python/syntaxes/Regular Expressions (Python).tmLanguage

-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<key>comment</key>
-	<string>Matches Python's regular expression syntax.</string>
-	<key>fileTypes</key>
-	<array>
-		<string>re</string>
-	</array>
-	<key>foldingStartMarker</key>
-	<string>(/\*|\{|\()</string>
-	<key>foldingStopMarker</key>
-	<string>(\*/|\}|\))</string>
-	<key>name</key>
-	<string>Regular Expressions (Python)</string>
-	<key>patterns</key>
-	<array>
-		<dict>
-			<key>match</key>
-			<string>\\[bBAZzG]|\^|\$</string>
-			<key>name</key>
-			<string>keyword.control.anchor.regexp</string>
-		</dict>
-		<dict>
-			<key>match</key>
-			<string>\\[1-9][0-9]?</string>
-			<key>name</key>
-			<string>keyword.other.back-reference.regexp</string>
-		</dict>
-		<dict>
-			<key>match</key>
-			<string>[?+*][?+]?|\{(\d+,\d+|\d+,|,\d+|\d+)\}\??</string>
-			<key>name</key>
-			<string>keyword.operator.quantifier.regexp</string>
-		</dict>
-		<dict>
-			<key>match</key>
-			<string>\|</string>
-			<key>name</key>
-			<string>keyword.operator.or.regexp</string>
-		</dict>
-		<dict>
-			<key>begin</key>
-			<string>\(\?\#</string>
-			<key>end</key>
-			<string>\)</string>
-			<key>name</key>
-			<string>comment.block.regexp</string>
-		</dict>
-		<dict>
-			<key>comment</key>
-			<string>We are restrictive in what we allow to go after the comment character to avoid false positives, since the availability of comments depend on regexp flags.</string>
-			<key>match</key>
-			<string>(?&lt;=^|\s)#\s[[a-zA-Z0-9,. \t?!-:][^\x{00}-\x{7F}]]*$</string>
-			<key>name</key>
-			<string>comment.line.number-sign.regexp</string>
-		</dict>
-		<dict>
-			<key>match</key>
-			<string>\(\?[iLmsux]+\)</string>
-			<key>name</key>
-			<string>keyword.other.option-toggle.regexp</string>
-		</dict>
-		<dict>
-			<key>match</key>
-			<string>(\()(\?P=([a-zA-Z_][a-zA-Z_0-9]*\w*))(\))</string>
-			<key>name</key>
-			<string>keyword.other.back-reference.named.regexp</string>
-		</dict>
-		<dict>
-			<key>begin</key>
-			<string>(\()((\?=)|(\?!)|(\?&lt;=)|(\?&lt;!))</string>
-			<key>beginCaptures</key>
-			<dict>
-				<key>1</key>
-				<dict>
-					<key>name</key>
-					<string>punctuation.definition.group.regexp</string>
-				</dict>
-				<key>2</key>
-				<dict>
-					<key>name</key>
-					<string>punctuation.definition.group.assertion.regexp</string>
-				</dict>
-				<key>3</key>
-				<dict>
-					<key>name</key>
-					<string>meta.assertion.look-ahead.regexp</string>
-				</dict>
-				<key>4</key>
-				<dict>
-					<key>name</key>
-					<string>meta.assertion.negative-look-ahead.regexp</string>
-				</dict>
-				<key>5</key>
-				<dict>
-					<key>name</key>
-					<string>meta.assertion.look-behind.regexp</string>
-				</dict>
-				<key>6</key>
-				<dict>
-					<key>name</key>
-					<string>meta.assertion.negative-look-behind.regexp</string>
-				</dict>
-			</dict>
-			<key>end</key>
-			<string>(\))</string>
-			<key>endCaptures</key>
-			<dict>
-				<key>1</key>
-				<dict>
-					<key>name</key>
-					<string>punctuation.definition.group.regexp</string>
-				</dict>
-			</dict>
-			<key>name</key>
-			<string>meta.group.assertion.regexp</string>
-			<key>patterns</key>
-			<array>
-				<dict>
-					<key>include</key>
-					<string>$self</string>
-				</dict>
-			</array>
-		</dict>
-		<dict>
-			<key>begin</key>
-			<string>(\()(\?\(([1-9][0-9]?|[a-zA-Z_][a-zA-Z_0-9]*)\))</string>
-			<key>beginCaptures</key>
-			<dict>
-				<key>1</key>
-				<dict>
-					<key>name</key>
-					<string>punctuation.definition.group.regexp</string>
-				</dict>
-				<key>2</key>
-				<dict>
-					<key>name</key>
-					<string>punctuation.definition.group.assertion.conditional.regexp</string>
-				</dict>
-				<key>3</key>
-				<dict>
-					<key>name</key>
-					<string>entity.name.section.back-reference.regexp</string>
-				</dict>
-			</dict>
-			<key>comment</key>
-			<string>we can make this more sophisticated to match the | character that separates yes-pattern from no-pattern, but it's not really necessary.</string>
-			<key>end</key>
-			<string>(\))</string>
-			<key>name</key>
-			<string>meta.group.assertion.conditional.regexp</string>
-			<key>patterns</key>
-			<array>
-				<dict>
-					<key>include</key>
-					<string>$self</string>
-				</dict>
-			</array>
-		</dict>
-		<dict>
-			<key>begin</key>
-			<string>(\()((\?P&lt;)([a-z]\w*)(&gt;)|(\?:))?</string>
-			<key>beginCaptures</key>
-			<dict>
-				<key>1</key>
-				<dict>
-					<key>name</key>
-					<string>punctuation.definition.group.regexp</string>
-				</dict>
-				<key>3</key>
-				<dict>
-					<key>name</key>
-					<string>punctuation.definition.group.capture.regexp</string>
-				</dict>
-				<key>4</key>
-				<dict>
-					<key>name</key>
-					<string>entity.name.section.group.regexp</string>
-				</dict>
-				<key>5</key>
-				<dict>
-					<key>name</key>
-					<string>punctuation.definition.group.capture.regexp</string>
-				</dict>
-				<key>6</key>
-				<dict>
-					<key>name</key>
-					<string>punctuation.definition.group.no-capture.regexp</string>
-				</dict>
-			</dict>
-			<key>end</key>
-			<string>(\))</string>
-			<key>endCaptures</key>
-			<dict>
-				<key>1</key>
-				<dict>
-					<key>name</key>
-					<string>punctuation.definition.group.regexp</string>
-				</dict>
-			</dict>
-			<key>name</key>
-			<string>meta.group.regexp</string>
-			<key>patterns</key>
-			<array>
-				<dict>
-					<key>include</key>
-					<string>$self</string>
-				</dict>
-			</array>
-		</dict>
-		<dict>
-			<key>include</key>
-			<string>#character-class</string>
-		</dict>
-	</array>
-	<key>repository</key>
-	<dict>
-		<key>character-class</key>
-		<dict>
-			<key>patterns</key>
-			<array>
-				<dict>
-					<key>match</key>
-					<string>\\[wWsSdDhH]|\.</string>
-					<key>name</key>
-					<string>constant.character.character-class.regexp</string>
-				</dict>
-				<dict>
-					<key>match</key>
-					<string>\\.</string>
-					<key>name</key>
-					<string>constant.character.escape.backslash.regexp</string>
-				</dict>
-				<dict>
-					<key>begin</key>
-					<string>(\[)(\^)?</string>
-					<key>beginCaptures</key>
-					<dict>
-						<key>1</key>
-						<dict>
-							<key>name</key>
-							<string>punctuation.definition.character-class.regexp</string>
-						</dict>
-						<key>2</key>
-						<dict>
-							<key>name</key>
-							<string>keyword.operator.negation.regexp</string>
-						</dict>
-					</dict>
-					<key>end</key>
-					<string>(\])</string>
-					<key>endCaptures</key>
-					<dict>
-						<key>1</key>
-						<dict>
-							<key>name</key>
-							<string>punctuation.definition.character-class.regexp</string>
-						</dict>
-					</dict>
-					<key>name</key>
-					<string>constant.other.character-class.set.regexp</string>
-					<key>patterns</key>
-					<array>
-						<dict>
-							<key>include</key>
-							<string>#character-class</string>
-						</dict>
-						<dict>
-							<key>captures</key>
-							<dict>
-								<key>2</key>
-								<dict>
-									<key>name</key>
-									<string>constant.character.escape.backslash.regexp</string>
-								</dict>
-								<key>4</key>
-								<dict>
-									<key>name</key>
-									<string>constant.character.escape.backslash.regexp</string>
-								</dict>
-							</dict>
-							<key>match</key>
-							<string>((\\.)|.)\-((\\.)|[^\]])</string>
-							<key>name</key>
-							<string>constant.other.character-class.range.regexp</string>
-						</dict>
-					</array>
-				</dict>
-			</array>
-		</dict>
-	</dict>
-	<key>scopeName</key>
-	<string>source.regexp.python</string>
-	<key>uuid</key>
-	<string>DD867ABF-1EC6-415D-B047-687F550A1D51</string>
-</dict>
-</plist>

extensions/python/test/colorize-fixtures/test.py

@@ -11,6 +11,14 @@ class Monkey:
 		eat(9.25)
 		return "Yum yum"
 
+	def some_func(a:
+					lambda x=None:
+					{key: val
+						for key, val in
+							(x if x is not None else [])
+					}=42):
+		pass
+
 if 1900 < year < 2100 and 1 <= month <= 12 \
    and 1 <= day <= 31 and 0 <= hour < 24 \
    and 0 <= minute < 60 and 0 <= second < 60:   # Looks like a valid date
@@ -64,3 +72,26 @@ while True:
 
 async with EXPR as VAR:
     BLOCK
+
+# Comments in dictionary items should be colorized accordingly
+my_dictionary = {
+    'foo':23, # this should be colorized as comment
+    'bar':"foobar" #this should be colorized as comment
+}
+
+# test raw strings
+text = r"""
+interval ``[1,2)`` leads to
+"""
+highlight_error = True
+
+# highlight doctests
+r'''Module docstring
+
+    Some text followed by code sample:
+    >>> for a in foo(2, b=1,
+    ...                 c=3):
+    ...   print(a)
+    0
+    1
+'''