Add support for including line feeds at the end of lines (microsoft/monaco-editor#2265)

src/vs/editor/common/model/textModelTokens.ts

@@ -359,7 +359,7 @@ export class TextModelTokenization extends Disposable {
 			const text = this._textModel.getLineContent(lineIndex + 1);
 			const lineStartState = this._tokenizationStateStore.getBeginState(lineIndex);
 
-			const r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, lineStartState!);
+			const r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, true, lineStartState!);
 			builder.add(lineIndex + 1, r.tokens);
 			this._tokenizationStateStore.setEndState(linesLength, lineIndex, r.endState);
 			lineIndex = this._tokenizationStateStore.invalidLineStartIndex - 1; // -1 because the outer loop increments it
@@ -410,13 +410,13 @@ export class TextModelTokenization extends Disposable {
 		const languageIdentifier = this._textModel.getLanguageIdentifier();
 		let state = initialState;
 		for (let i = fakeLines.length - 1; i >= 0; i--) {
-			let r = safeTokenize(languageIdentifier, this._tokenizationSupport, fakeLines[i], state);
+			let r = safeTokenize(languageIdentifier, this._tokenizationSupport, fakeLines[i], false, state);
 			state = r.endState;
 		}
 
 		for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) {
 			let text = this._textModel.getLineContent(lineNumber);
-			let r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, state);
+			let r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, true, state);
 			builder.add(lineNumber, r.tokens);
 			this._tokenizationStateStore.setFakeTokens(lineNumber - 1);
 			state = r.endState;
@@ -443,12 +443,12 @@ function initializeTokenization(textModel: TextModel): [ITokenizationSupport | n
 	return [tokenizationSupport, initialState];
 }
 
-function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, state: IState): TokenizationResult2 {
+function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, hasEOL: boolean, state: IState): TokenizationResult2 {
 	let r: TokenizationResult2 | null = null;
 
 	if (tokenizationSupport) {
 		try {
-			r = tokenizationSupport.tokenize2(text, state.clone(), 0);
+			r = tokenizationSupport.tokenize2(text, hasEOL, state.clone(), 0);
 		} catch (e) {
 			onUnexpectedError(e);
 		}

src/vs/editor/common/modes.ts

@@ -211,9 +211,9 @@ export interface ITokenizationSupport {
 	getInitialState(): IState;
 
 	// add offsetDelta to each of the returned indices
-	tokenize(line: string, state: IState, offsetDelta: number): TokenizationResult;
+	tokenize(line: string, hasEOL: boolean, state: IState, offsetDelta: number): TokenizationResult;
 
-	tokenize2(line: string, state: IState, offsetDelta: number): TokenizationResult2;
+	tokenize2(line: string, hasEOL: boolean, state: IState, offsetDelta: number): TokenizationResult2;
 }
 
 /**

src/vs/editor/common/modes/textToHtmlTokenizer.ts

@@ -12,12 +12,12 @@ import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 
 export interface IReducedTokenizationSupport {
 	getInitialState(): IState;
-	tokenize2(line: string, state: IState, offsetDelta: number): TokenizationResult2;
+	tokenize2(line: string, hasEOL: boolean, state: IState, offsetDelta: number): TokenizationResult2;
 }
 
 const fallback: IReducedTokenizationSupport = {
 	getInitialState: () => NULL_STATE,
-	tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
+	tokenize2: (buffer: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
 };
 
 export function tokenizeToString(text: string, tokenizationSupport: IReducedTokenizationSupport = fallback): string {
@@ -110,7 +110,7 @@ function _tokenizeToString(text: string, tokenizationSupport: IReducedTokenizati
 			result += `<br/>`;
 		}
 
-		let tokenizationResult = tokenizationSupport.tokenize2(line, currentState, 0);
+		let tokenizationResult = tokenizationSupport.tokenize2(line, true, currentState, 0);
 		LineTokens.convertToEndOffset(tokenizationResult.tokens, line.length);
 		let lineTokens = new LineTokens(tokenizationResult.tokens, line);
 		let viewLineTokens = lineTokens.inflate();

src/vs/editor/contrib/comment/test/lineCommentCommand.test.ts

@@ -1083,7 +1083,7 @@ suite('Editor Contrib - Line Comment in mixed modes', () => {
 				tokenize: () => {
 					throw new Error('not implemented');
 				},
-				tokenize2: (line: string, state: modes.IState): TokenizationResult2 => {
+				tokenize2: (line: string, hasEOL: boolean, state: modes.IState): TokenizationResult2 => {
 					let languageId = (/^  /.test(line) ? INNER_LANGUAGE_ID : OUTER_LANGUAGE_ID);
 
 					let tokens = new Uint32Array(1 << 1);

src/vs/editor/contrib/suggest/test/suggestModel.test.ts

@@ -71,7 +71,7 @@ suite('SuggestModel - Context', function () {
 			this._register(TokenizationRegistry.register(this.getLanguageIdentifier().language, {
 				getInitialState: (): IState => NULL_STATE,
 				tokenize: undefined!,
-				tokenize2: (line: string, state: IState): TokenizationResult2 => {
+				tokenize2: (line: string, hasEOL: boolean, state: IState): TokenizationResult2 => {
 					const tokensArr: number[] = [];
 					let prevLanguageId: LanguageIdentifier | undefined = undefined;
 					for (let i = 0; i < line.length; i++) {

src/vs/editor/standalone/browser/colorizer.ts

@@ -222,7 +222,7 @@ function _actualColorize(lines: string[], tabSize: number, tokenizationSupport:
 
 	for (let i = 0, length = lines.length; i < length; i++) {
 		let line = lines[i];
-		let tokenizeResult = tokenizationSupport.tokenize2(line, state, 0);
+		let tokenizeResult = tokenizationSupport.tokenize2(line, true, state, 0);
 		LineTokens.convertToEndOffset(tokenizeResult.tokens, line.length);
 		let lineTokens = new LineTokens(tokenizeResult.tokens, line);
 		const isBasicASCII = ViewLineRenderingData.isBasicASCII(line, /* check for basic ASCII */true);

src/vs/editor/standalone/browser/inspectTokens/inspectTokens.ts

@@ -137,8 +137,8 @@ function getSafeTokenizationSupport(languageIdentifier: LanguageIdentifier): ITo
 	}
 	return {
 		getInitialState: () => NULL_STATE,
-		tokenize: (line: string, state: IState, deltaOffset: number) => nullTokenize(languageIdentifier.language, line, state, deltaOffset),
-		tokenize2: (line: string, state: IState, deltaOffset: number) => nullTokenize2(languageIdentifier.id, line, state, deltaOffset)
+		tokenize: (line: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenize(languageIdentifier.language, line, state, deltaOffset),
+		tokenize2: (line: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenize2(languageIdentifier.id, line, state, deltaOffset)
 	};
 }
 
@@ -293,8 +293,8 @@ class InspectTokensWidget extends Disposable implements IContentWidget {
 	private _getTokensAtLine(lineNumber: number): ICompleteLineTokenization {
 		let stateBeforeLine = this._getStateBeforeLine(lineNumber);
 
-		let tokenizationResult1 = this._tokenizationSupport.tokenize(this._model.getLineContent(lineNumber), stateBeforeLine, 0);
-		let tokenizationResult2 = this._tokenizationSupport.tokenize2(this._model.getLineContent(lineNumber), stateBeforeLine, 0);
+		let tokenizationResult1 = this._tokenizationSupport.tokenize(this._model.getLineContent(lineNumber), true, stateBeforeLine, 0);
+		let tokenizationResult2 = this._tokenizationSupport.tokenize2(this._model.getLineContent(lineNumber), true, stateBeforeLine, 0);
 
 		return {
 			startState: stateBeforeLine,
@@ -308,7 +308,7 @@ class InspectTokensWidget extends Disposable implements IContentWidget {
 		let state: IState = this._tokenizationSupport.getInitialState();
 
 		for (let i = 1; i < lineNumber; i++) {
-			let tokenizationResult = this._tokenizationSupport.tokenize(this._model.getLineContent(i), state, 0);
+			let tokenizationResult = this._tokenizationSupport.tokenize(this._model.getLineContent(i), true, state, 0);
 			state = tokenizationResult.endState;
 		}
 

src/vs/editor/standalone/browser/standaloneEditor.ts

@@ -276,7 +276,7 @@ function getSafeTokenizationSupport(language: string): Omit<modes.ITokenizationS
 	}
 	return {
 		getInitialState: () => NULL_STATE,
-		tokenize: (line: string, state: modes.IState, deltaOffset: number) => nullTokenize(language, line, state, deltaOffset)
+		tokenize: (line: string, hasEOL: boolean, state: modes.IState, deltaOffset: number) => nullTokenize(language, line, state, deltaOffset)
 	};
 }
 
@@ -294,7 +294,7 @@ export function tokenize(text: string, languageId: string): Token[][] {
 	let state = tokenizationSupport.getInitialState();
 	for (let i = 0, len = lines.length; i < len; i++) {
 		let line = lines[i];
-		let tokenizationResult = tokenizationSupport.tokenize(line, state, 0);
+		let tokenizationResult = tokenizationSupport.tokenize(line, true, state, 0);
 
 		result[i] = tokenizationResult.tokens;
 		state = tokenizationResult.endState;

src/vs/editor/standalone/browser/standaloneLanguages.ts

@@ -88,14 +88,14 @@ export class EncodedTokenizationSupport2Adapter implements modes.ITokenizationSu
 		return this._actual.getInitialState();
 	}
 
-	public tokenize(line: string, state: modes.IState, offsetDelta: number): TokenizationResult {
+	public tokenize(line: string, hasEOL: boolean, state: modes.IState, offsetDelta: number): TokenizationResult {
 		if (typeof this._actual.tokenize === 'function') {
 			return TokenizationSupport2Adapter.adaptTokenize(this._languageIdentifier.language, <{ tokenize(line: string, state: modes.IState): ILineTokens; }>this._actual, line, state, offsetDelta);
 		}
 		throw new Error('Not supported!');
 	}
 
-	public tokenize2(line: string, state: modes.IState): TokenizationResult2 {
+	public tokenize2(line: string, hasEOL: boolean, state: modes.IState): TokenizationResult2 {
 		let result = this._actual.tokenizeEncoded(line, state);
 		return new TokenizationResult2(result.tokens, result.endState);
 	}
@@ -158,7 +158,7 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
 		return new TokenizationResult(tokens, endState);
 	}
 
-	public tokenize(line: string, state: modes.IState, offsetDelta: number): TokenizationResult {
+	public tokenize(line: string, hasEOL: boolean, state: modes.IState, offsetDelta: number): TokenizationResult {
 		return TokenizationSupport2Adapter.adaptTokenize(this._languageIdentifier.language, this._actual, line, state, offsetDelta);
 	}
 
@@ -200,7 +200,7 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
 		return actualResult;
 	}
 
-	public tokenize2(line: string, state: modes.IState, offsetDelta: number): TokenizationResult2 {
+	public tokenize2(line: string, hasEOL: boolean, state: modes.IState, offsetDelta: number): TokenizationResult2 {
 		let actualResult = this._actual.tokenize(line, state);
 		let tokens = this._toBinaryTokens(actualResult.tokens, offsetDelta);
 

src/vs/editor/standalone/common/monarch/monarchCommon.ts

@@ -22,6 +22,7 @@ export const enum MonarchBracket {
 
 export interface ILexerMin {
 	languageId: string;
+	includeLF: boolean;
 	noThrow: boolean;
 	ignoreCase: boolean;
 	unicode: boolean;

src/vs/editor/standalone/common/monarch/monarchCompile.ts

@@ -395,6 +395,7 @@ export function compile(languageId: string, json: IMonarchLanguage): monarchComm
 	// Create our lexer
 	let lexer: monarchCommon.ILexer = <monarchCommon.ILexer>{};
 	lexer.languageId = languageId;
+	lexer.includeLF = bool(json.includeLF, false);
 	lexer.noThrow = false; // raise exceptions during compilation
 	lexer.maxStack = 100;
 
@@ -411,6 +412,7 @@ export function compile(languageId: string, json: IMonarchLanguage): monarchComm
 	// For calling compileAction later on
 	let lexerMin: monarchCommon.ILexerMin = <any>json;
 	lexerMin.languageId = languageId;
+	lexerMin.includeLF = lexer.includeLF;
 	lexerMin.ignoreCase = lexer.ignoreCase;
 	lexerMin.unicode = lexer.unicode;
 	lexerMin.noThrow = lexer.noThrow;

src/vs/editor/standalone/common/monarch/monarchLexer.ts

@@ -231,7 +231,7 @@ class MonarchLineState implements modes.IState {
 interface IMonarchTokensCollector {
 	enterMode(startOffset: number, modeId: string): void;
 	emit(startOffset: number, type: string): void;
-	nestedModeTokenize(embeddedModeLine: string, embeddedModeData: EmbeddedModeData, offsetDelta: number): modes.IState;
+	nestedModeTokenize(embeddedModeLine: string, hasEOL: boolean, embeddedModeData: EmbeddedModeData, offsetDelta: number): modes.IState;
 }
 
 class MonarchClassicTokensCollector implements IMonarchTokensCollector {
@@ -261,7 +261,7 @@ class MonarchClassicTokensCollector implements IMonarchTokensCollector {
 		this._tokens.push(new Token(startOffset, type, this._language!));
 	}
 
-	public nestedModeTokenize(embeddedModeLine: string, embeddedModeData: EmbeddedModeData, offsetDelta: number): modes.IState {
+	public nestedModeTokenize(embeddedModeLine: string, hasEOL: boolean, embeddedModeData: EmbeddedModeData, offsetDelta: number): modes.IState {
 		const nestedModeId = embeddedModeData.modeId;
 		const embeddedModeState = embeddedModeData.state;
 
@@ -272,7 +272,7 @@ class MonarchClassicTokensCollector implements IMonarchTokensCollector {
 			return embeddedModeState;
 		}
 
-		let nestedResult = nestedModeTokenizationSupport.tokenize(embeddedModeLine, embeddedModeState, offsetDelta);
+		let nestedResult = nestedModeTokenizationSupport.tokenize(embeddedModeLine, hasEOL, embeddedModeState, offsetDelta);
 		this._tokens = this._tokens.concat(nestedResult.tokens);
 		this._lastTokenType = null;
 		this._lastTokenLanguage = null;
@@ -345,7 +345,7 @@ class MonarchModernTokensCollector implements IMonarchTokensCollector {
 		return result;
 	}
 
-	public nestedModeTokenize(embeddedModeLine: string, embeddedModeData: EmbeddedModeData, offsetDelta: number): modes.IState {
+	public nestedModeTokenize(embeddedModeLine: string, hasEOL: boolean, embeddedModeData: EmbeddedModeData, offsetDelta: number): modes.IState {
 		const nestedModeId = embeddedModeData.modeId;
 		const embeddedModeState = embeddedModeData.state;
 
@@ -356,7 +356,7 @@ class MonarchModernTokensCollector implements IMonarchTokensCollector {
 			return embeddedModeState;
 		}
 
-		let nestedResult = nestedModeTokenizationSupport.tokenize2(embeddedModeLine, embeddedModeState, offsetDelta);
+		let nestedResult = nestedModeTokenizationSupport.tokenize2(embeddedModeLine, hasEOL, embeddedModeState, offsetDelta);
 		this._prependTokens = MonarchModernTokensCollector._merge(this._prependTokens, this._tokens, nestedResult.tokens);
 		this._tokens = [];
 		this._currentLanguageId = 0;
@@ -456,23 +456,23 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
 		return MonarchLineStateFactory.create(rootState, null);
 	}
 
-	public tokenize(line: string, lineState: modes.IState, offsetDelta: number): TokenizationResult {
+	public tokenize(line: string, hasEOL: boolean, lineState: modes.IState, offsetDelta: number): TokenizationResult {
 		let tokensCollector = new MonarchClassicTokensCollector();
-		let endLineState = this._tokenize(line, <MonarchLineState>lineState, offsetDelta, tokensCollector);
+		let endLineState = this._tokenize(line, hasEOL, <MonarchLineState>lineState, offsetDelta, tokensCollector);
 		return tokensCollector.finalize(endLineState);
 	}
 
-	public tokenize2(line: string, lineState: modes.IState, offsetDelta: number): TokenizationResult2 {
+	public tokenize2(line: string, hasEOL: boolean, lineState: modes.IState, offsetDelta: number): TokenizationResult2 {
 		let tokensCollector = new MonarchModernTokensCollector(this._modeService, this._standaloneThemeService.getColorTheme().tokenTheme);
-		let endLineState = this._tokenize(line, <MonarchLineState>lineState, offsetDelta, tokensCollector);
+		let endLineState = this._tokenize(line, hasEOL, <MonarchLineState>lineState, offsetDelta, tokensCollector);
 		return tokensCollector.finalize(endLineState);
 	}
 
-	private _tokenize(line: string, lineState: MonarchLineState, offsetDelta: number, collector: IMonarchTokensCollector): MonarchLineState {
+	private _tokenize(line: string, hasEOL: boolean, lineState: MonarchLineState, offsetDelta: number, collector: IMonarchTokensCollector): MonarchLineState {
 		if (lineState.embeddedModeData) {
-			return this._nestedTokenize(line, lineState, offsetDelta, collector);
+			return this._nestedTokenize(line, hasEOL, lineState, offsetDelta, collector);
 		} else {
-			return this._myTokenize(line, lineState, offsetDelta, collector);
+			return this._myTokenize(line, hasEOL, lineState, offsetDelta, collector);
 		}
 	}
 
@@ -518,24 +518,24 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
 		return popOffset;
 	}
 
-	private _nestedTokenize(line: string, lineState: MonarchLineState, offsetDelta: number, tokensCollector: IMonarchTokensCollector): MonarchLineState {
+	private _nestedTokenize(line: string, hasEOL: boolean, lineState: MonarchLineState, offsetDelta: number, tokensCollector: IMonarchTokensCollector): MonarchLineState {
 
 		let popOffset = this._findLeavingNestedModeOffset(line, lineState);
 
 		if (popOffset === -1) {
 			// tokenization will not leave nested mode
-			let nestedEndState = tokensCollector.nestedModeTokenize(line, lineState.embeddedModeData!, offsetDelta);
+			let nestedEndState = tokensCollector.nestedModeTokenize(line, hasEOL, lineState.embeddedModeData!, offsetDelta);
 			return MonarchLineStateFactory.create(lineState.stack, new EmbeddedModeData(lineState.embeddedModeData!.modeId, nestedEndState));
 		}
 
 		let nestedModeLine = line.substring(0, popOffset);
 		if (nestedModeLine.length > 0) {
 			// tokenize with the nested mode
-			tokensCollector.nestedModeTokenize(nestedModeLine, lineState.embeddedModeData!, offsetDelta);
+			tokensCollector.nestedModeTokenize(nestedModeLine, false, lineState.embeddedModeData!, offsetDelta);
 		}
 
 		let restOfTheLine = line.substring(popOffset);
-		return this._myTokenize(restOfTheLine, lineState, offsetDelta + popOffset, tokensCollector);
+		return this._myTokenize(restOfTheLine, hasEOL, lineState, offsetDelta + popOffset, tokensCollector);
 	}
 
 	private _safeRuleName(rule: monarchCommon.IRule | null): string {
@@ -545,9 +545,11 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
 		return '(unknown)';
 	}
 
-	private _myTokenize(line: string, lineState: MonarchLineState, offsetDelta: number, tokensCollector: IMonarchTokensCollector): MonarchLineState {
+	private _myTokenize(lineWithoutLF: string, hasEOL: boolean, lineState: MonarchLineState, offsetDelta: number, tokensCollector: IMonarchTokensCollector): MonarchLineState {
 		tokensCollector.enterMode(offsetDelta, this._modeId);
 
+		const lineWithoutLFLength = lineWithoutLF.length;
+		const line = (hasEOL && this._lexer.includeLF ? lineWithoutLF + '\n' : lineWithoutLF);
 		const lineLength = line.length;
 
 		let embeddedModeData = lineState.embeddedModeData;
@@ -752,8 +754,8 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
 
 				if (pos < lineLength) {
 					// there is content from the embedded mode on this line
-					const restOfLine = line.substr(pos);
-					return this._nestedTokenize(restOfLine, MonarchLineStateFactory.create(stack, embeddedModeData), offsetDelta + pos, tokensCollector);
+					const restOfLine = lineWithoutLF.substr(pos);
+					return this._nestedTokenize(restOfLine, hasEOL, MonarchLineStateFactory.create(stack, embeddedModeData), offsetDelta + pos, tokensCollector);
 				} else {
 					return MonarchLineStateFactory.create(stack, embeddedModeData);
 				}
@@ -831,7 +833,9 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
 					tokenType = monarchCommon.sanitize(token);
 				}
 
-				tokensCollector.emit(pos0 + offsetDelta, tokenType);
+				if (pos0 < lineWithoutLFLength) {
+					tokensCollector.emit(pos0 + offsetDelta, tokenType);
+				}
 			}
 
 			if (enteringEmbeddedMode !== null) {

src/vs/editor/standalone/common/monarch/monarchTypes.ts

@@ -41,6 +41,11 @@ export interface IMonarchLanguage {
 	 * attach this to every token class (by default '.' + name)
 	 */
 	tokenPostfix?: string;
+	/**
+	 * include line feeds (in the form of a \n character) at the end of lines
+	 * Defaults to false
+	 */
+	includeLF?: boolean;
 }
 
 /**

src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts

@@ -108,10 +108,10 @@ suite('TokenizationSupport2Adapter', () => {
 
 		const adapter = new TokenizationSupport2Adapter(new MockThemeService(), languageIdentifier, new BadTokensProvider());
 
-		const actualClassicTokens = adapter.tokenize('whatever', MockState.INSTANCE, offsetDelta);
+		const actualClassicTokens = adapter.tokenize('whatever', true, MockState.INSTANCE, offsetDelta);
 		assert.deepEqual(actualClassicTokens.tokens, expectedClassicTokens);
 
-		const actualModernTokens = adapter.tokenize2('whatever', MockState.INSTANCE, offsetDelta);
+		const actualModernTokens = adapter.tokenize2('whatever', true, MockState.INSTANCE, offsetDelta);
 		const modernTokens: number[] = [];
 		for (let i = 0; i < actualModernTokens.tokens.length; i++) {
 			modernTokens[i] = actualModernTokens.tokens[i];

src/vs/editor/standalone/test/monarch/monarch.test.ts

@@ -68,7 +68,7 @@ suite('Monarch', () => {
 		const actualTokens: Token[][] = [];
 		let state = tokenizer.getInitialState();
 		for (const line of lines) {
-			const result = tokenizer.tokenize(line, state, 0);
+			const result = tokenizer.tokenize(line, true, state, 0);
 			actualTokens.push(result.tokens);
 			state = result.endState;
 		}
@@ -143,7 +143,7 @@ suite('Monarch', () => {
 		const actualTokens: Token[][] = [];
 		let state = tokenizer.getInitialState();
 		for (const line of lines) {
-			const result = tokenizer.tokenize(line, state, 0);
+			const result = tokenizer.tokenize(line, true, state, 0);
 			actualTokens.push(result.tokens);
 			state = result.endState;
 		}
@@ -164,4 +164,58 @@ suite('Monarch', () => {
 		]);
 
 	});
+
+	test('microsoft/monaco-editor#2265: Exit a state at end of line', () => {
+		const modeService = new ModeServiceImpl();
+		const tokenizer = createMonarchTokenizer(modeService, 'test', {
+			includeLF: true,
+			tokenizer: {
+				root: [
+					[/^\*/, '', '@inner'],
+					[/\:\*/, '', '@inner'],
+					[/[^*:]+/, 'string'],
+					[/[*:]/, 'string']
+				],
+				inner: [
+					[/\n/, '', '@pop'],
+					[/\d+/, 'number'],
+					[/[^\d]+/, '']
+				]
+			}
+		});
+
+		const lines = [
+			`PRINT 10 * 20`,
+			`*FX200, 3`,
+			`PRINT 2*3:*FX200, 3`
+		];
+
+		const actualTokens: Token[][] = [];
+		let state = tokenizer.getInitialState();
+		for (const line of lines) {
+			const result = tokenizer.tokenize(line, true, state, 0);
+			actualTokens.push(result.tokens);
+			state = result.endState;
+		}
+
+		assert.deepStrictEqual(actualTokens, [
+			[
+				new Token(0, 'string.test', 'test'),
+			],
+			[
+				new Token(0, '', 'test'),
+				new Token(3, 'number.test', 'test'),
+				new Token(6, '', 'test'),
+				new Token(8, 'number.test', 'test'),
+			],
+			[
+				new Token(0, 'string.test', 'test'),
+				new Token(9, '', 'test'),
+				new Token(13, 'number.test', 'test'),
+				new Token(16, '', 'test'),
+				new Token(18, 'number.test', 'test'),
+			]
+		]);
+	});
+
 });

src/vs/editor/test/browser/controller/cursor.test.ts

@@ -2406,7 +2406,7 @@ suite('Editor Controller - Regression tests', () => {
 		const tokenizationSupport: ITokenizationSupport = {
 			getInitialState: () => NULL_STATE,
 			tokenize: undefined!,
-			tokenize2: (line: string, state: IState): TokenizationResult2 => {
+			tokenize2: (line: string, hasEOL: boolean, state: IState): TokenizationResult2 => {
 				return new TokenizationResult2(new Uint32Array(0), state);
 			}
 		};

src/vs/editor/test/common/model/model.modes.test.ts

@@ -28,7 +28,7 @@ suite('Editor Model - Model Modes 1', () => {
 	const tokenizationSupport: modes.ITokenizationSupport = {
 		getInitialState: () => NULL_STATE,
 		tokenize: undefined!,
-		tokenize2: (line: string, state: modes.IState): TokenizationResult2 => {
+		tokenize2: (line: string, hasEOL: boolean, state: modes.IState): TokenizationResult2 => {
 			calledFor.push(line.charAt(0));
 			return new TokenizationResult2(new Uint32Array(0), state);
 		}
@@ -181,7 +181,7 @@ suite('Editor Model - Model Modes 2', () => {
 	const tokenizationSupport: modes.ITokenizationSupport = {
 		getInitialState: () => new ModelState2(''),
 		tokenize: undefined!,
-		tokenize2: (line: string, state: modes.IState): TokenizationResult2 => {
+		tokenize2: (line: string, hasEOL: boolean, state: modes.IState): TokenizationResult2 => {
 			calledFor.push(line);
 			(<ModelState2>state).prevLineContent = line;
 			return new TokenizationResult2(new Uint32Array(0), state);

src/vs/editor/test/common/model/model.test.ts

@@ -389,7 +389,7 @@ suite('Editor Model - Words', () => {
 			this._register(TokenizationRegistry.register(this.getLanguageIdentifier().language, {
 				getInitialState: (): IState => NULL_STATE,
 				tokenize: undefined!,
-				tokenize2: (line: string, state: IState): TokenizationResult2 => {
+				tokenize2: (line: string, hasEOL: boolean, state: IState): TokenizationResult2 => {
 					const tokensArr: number[] = [];
 					let prevLanguageId: LanguageIdentifier | undefined = undefined;
 					for (let i = 0; i < line.length; i++) {

src/vs/editor/test/common/model/textModelWithTokens.test.ts

@@ -351,7 +351,7 @@ suite('TextModelWithTokens', () => {
 		const tokenizationSupport: ITokenizationSupport = {
 			getInitialState: () => NULL_STATE,
 			tokenize: undefined!,
-			tokenize2: (line, state) => {
+			tokenize2: (line, hasEOL, state) => {
 				switch (line) {
 					case 'function hello() {': {
 						const tokens = new Uint32Array([
@@ -446,7 +446,7 @@ suite('TextModelWithTokens regression tests', () => {
 		const tokenizationSupport: ITokenizationSupport = {
 			getInitialState: () => NULL_STATE,
 			tokenize: undefined!,
-			tokenize2: (line, state) => {
+			tokenize2: (line, hasEOL, state) => {
 				let myId = ++_tokenId;
 				let tokens = new Uint32Array(2);
 				tokens[0] = 0;
@@ -550,7 +550,7 @@ suite('TextModelWithTokens regression tests', () => {
 		const tokenizationSupport: ITokenizationSupport = {
 			getInitialState: () => NULL_STATE,
 			tokenize: undefined!,
-			tokenize2: (line, state) => {
+			tokenize2: (line, hasEOL, state) => {
 				let tokens = new Uint32Array(2);
 				tokens[0] = 0;
 				tokens[1] = (

src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts

@@ -287,7 +287,7 @@ class Mode extends MockMode {
 		this._register(TokenizationRegistry.register(this.getId(), {
 			getInitialState: (): IState => null!,
 			tokenize: undefined!,
-			tokenize2: (line: string, state: IState): TokenizationResult2 => {
+			tokenize2: (line: string, hasEOL: boolean, state: IState): TokenizationResult2 => {
 				let tokensArr: number[] = [];
 				let prevColor: ColorId = -1;
 				for (let i = 0; i < line.length; i++) {

src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts

@@ -333,7 +333,7 @@ suite('SplitLinesCollection', () => {
 		const tokenizationSupport: modes.ITokenizationSupport = {
 			getInitialState: () => NULL_STATE,
 			tokenize: undefined!,
-			tokenize2: (line: string, state: modes.IState): TokenizationResult2 => {
+			tokenize2: (line: string, hasEOL: boolean, state: modes.IState): TokenizationResult2 => {
 				let tokens = _tokens[_lineIndex++];
 
 				let result = new Uint32Array(2 * tokens.length);

src/vs/monaco.d.ts

@@ -6449,6 +6449,11 @@ declare namespace monaco.languages {
 		 * attach this to every token class (by default '.' + name)
 		 */
 		tokenPostfix?: string;
+		/**
+		 * include line feeds (in the form of a \n character) at the end of lines
+		 * Defaults to false
+		 */
+		includeLF?: boolean;
 	}
 
 	/**

src/vs/workbench/services/textMate/browser/abstractTextMateService.ts

@@ -445,11 +445,11 @@ class TMTokenizationSupport implements ITokenizationSupport {
 		return this._actual.getInitialState();
 	}
 
-	tokenize(line: string, state: IState, offsetDelta: number): TokenizationResult {
+	tokenize(line: string, hasEOL: boolean, state: IState, offsetDelta: number): TokenizationResult {
 		throw new Error('Not supported!');
 	}
 
-	tokenize2(line: string, state: StackElement, offsetDelta: number): TokenizationResult2 {
+	tokenize2(line: string, hasEOL: boolean, state: StackElement, offsetDelta: number): TokenizationResult2 {
 		if (offsetDelta !== 0) {
 			throw new Error('Unexpected: offsetDelta should be 0.');
 		}