New tokens format

build/monaco/monaco.d.ts.recipe

@@ -63,14 +63,23 @@ export interface ICommandHandler {
 #include(vs/platform/markers/common/markers): IMarkerData
 #include(vs/editor/browser/standalone/colorizer): IColorizerOptions, IColorizerElementOptions
 #include(vs/base/common/scrollable): ScrollbarVisibility
-#includeAll(vs/editor/common/editorCommon;IMode=>languages.IMode): IPosition, IRange, ISelection, SelectionDirection, IScrollEvent
+#includeAll(vs/editor/common/editorCommon;IMode=>languages.IMode;LanguageIdentifier=>languages.LanguageIdentifier): IPosition, IRange, ISelection, SelectionDirection, IScrollEvent
 #includeAll(vs/editor/browser/editorBrowser;editorCommon.=>):
 }
 
 declare module monaco.languages {
 
 #includeAll(vs/editor/browser/standalone/standaloneLanguages;modes.=>;editorCommon.=>editor.;IMarkerData=>editor.IMarkerData):
 #includeAll(vs/editor/common/modes/languageConfiguration):
+/**
+ * An identifier for a registered language.
+ */
+export class LanguageIdentifier {
+	public readonly sid: string;
+	public readonly iid: number;
+
+	constructor(sid: string, iid: number);
+}
 #includeAll(vs/editor/common/modes;editorCommon.IRange=>IRange;editorCommon.IPosition=>IPosition;editorCommon.=>editor.;IToken2=>IToken;ILineTokens2=>ILineTokens):
 #include(vs/editor/common/services/modeService): ILanguageExtensionPoint
 #includeAll(vs/editor/common/modes/monarch/monarchTypes):

src/vs/editor/browser/services/standaloneColorServiceImpl.ts

@@ -0,0 +1,43 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import Event, { Emitter } from 'vs/base/common/event';
+import { Theme } from 'vs/editor/common/modes/supports/tokenization';
+import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
+import { vs } from 'vs/editor/common/standalone/themes';
+import * as dom from 'vs/base/browser/dom';
+
+export class StandaloneColorServiceImpl implements IStandaloneColorService {
+
+	_serviceBrand: any;
+
+	private _onThemeChanged: Emitter<void> = new Emitter<void>();
+	public onThemeChanged: Event<void> = this._onThemeChanged.event;
+
+	private _theme: Theme;
+	private _styleElement: HTMLStyleElement;
+
+	constructor() {
+		this._theme = Theme.createFromRawTheme(vs);
+		this._styleElement = dom.createStyleSheet();
+
+		let colorMap = this._theme.getColorMap();
+		let rules: string[] = [];
+		for (let i = 0, len = colorMap.length; i < len; i++) {
+			let color = colorMap[i];
+			rules[i] = `.mtk${i} { color: #${color}; }`;
+		}
+		rules.push('.mtki { font-style: italic; }');
+		rules.push('.mtkb { font-weight: bold; }');
+		rules.push('.mtku { text-decoration: underline; }');
+
+		this._styleElement.innerHTML = rules.join('\n');
+	}
+
+	public getTheme(): Theme {
+		return this._theme;
+	}
+}

src/vs/editor/browser/standalone/colorizer.ts

@@ -12,6 +12,7 @@ import { IModeService } from 'vs/editor/common/services/modeService';
 import { renderLine, RenderLineInput } from 'vs/editor/common/viewLayout/viewLineRenderer';
 import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 import { LineParts } from 'vs/editor/common/core/lineParts';
+import { LineTokens } from 'vs/editor/common/core/lineTokens';
 import * as strings from 'vs/base/common/strings';
 
 export interface IColorizerOptions {
@@ -141,20 +142,20 @@ function _fakeColorize(lines: string[], tabSize: number): string {
 function _actualColorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport): string {
 	let html: string[] = [];
 	let state = tokenizationSupport.getInitialState();
+	let colorMap = TokenizationRegistry.getColorMap();
 
 	for (let i = 0, length = lines.length; i < length; i++) {
 		let line = lines[i];
-
-		let tokenizeResult = tokenizationSupport.tokenize(line, state, 0);
-
+		let tokenizeResult = tokenizationSupport.tokenize3(line, state, 0);
+		let lineTokens = new LineTokens(colorMap, tokenizeResult.tokens, line);
 		let renderResult = renderLine(new RenderLineInput(
 			line,
 			tabSize,
 			0,
 			-1,
 			'none',
 			false,
-			new LineParts(tokenizeResult.tokens.map(t => new ViewLineToken(t.startIndex, t.type)), line.length + 1)
+			new LineParts(lineTokens.inflate(), line.length + 1)
 		));
 
 		html = html.concat(renderResult.output);

src/vs/editor/browser/standalone/standaloneLanguages.ts

@@ -23,7 +23,10 @@ import { compile } from 'vs/editor/common/modes/monarch/monarchCompile';
 import { createTokenizationSupport } from 'vs/editor/common/modes/monarch/monarchLexer';
 import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
 import { IMarkerData } from 'vs/platform/markers/common/markers';
-import { TokenizationSupport2Adapter } from 'vs/editor/common/services/modeServiceImpl';
+import { Token } from 'vs/editor/common/core/token';
+import { ModeTransition } from 'vs/editor/common/core/modeTransition';
+import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
+
 /**
  * Register information about a new language.
  */
@@ -61,14 +64,101 @@ export function onLanguage(languageId: string, callback: () => void): IDisposabl
  * Set the editing configuration for a language.
  */
 export function setLanguageConfiguration(languageId: string, configuration: LanguageConfiguration): IDisposable {
-	return LanguageConfigurationRegistry.register(languageId, configuration);
+	let languageIdentifier = StaticServices.modeService.get().getLanguageIdentifier(languageId);
+	if (!languageIdentifier) {
+		throw new Error(`Cannot set configuration for unknown language ${languageId}`);
+	}
+	return LanguageConfigurationRegistry.register(languageIdentifier, configuration);
+}
+
+/**
+ * @internal
+ */
+export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
+
+	private readonly _standaloneColorService: IStandaloneColorService;
+	private readonly _languageIdentifier: modes.LanguageIdentifier;
+	private readonly _actual: modes.TokensProvider;
+
+	constructor(standaloneColorService: IStandaloneColorService, languageIdentifier: modes.LanguageIdentifier, actual: modes.TokensProvider) {
+		this._standaloneColorService = standaloneColorService;
+		this._languageIdentifier = languageIdentifier;
+		this._actual = actual;
+	}
+
+	public getInitialState(): modes.IState {
+		return this._actual.getInitialState();
+	}
+
+	private _toClassicTokens(tokens: modes.IToken2[], offsetDelta: number): Token[] {
+		let result: Token[] = [];
+		for (let i = 0, len = tokens.length; i < len; i++) {
+			let t = tokens[i];
+			result[i] = new Token(t.startIndex + offsetDelta, t.scopes);
+		}
+		return result;
+	}
+
+	public tokenize(line: string, state: modes.IState, offsetDelta: number): modes.ILineTokens {
+		let actualResult = this._actual.tokenize(line, state);
+		let tokens = this._toClassicTokens(actualResult.tokens, offsetDelta);
+
+		let endState: modes.IState;
+		// try to save an object if possible
+		if (actualResult.endState.equals(state)) {
+			endState = state;
+		} else {
+			endState = actualResult.endState;
+		}
+
+		return {
+			tokens: tokens,
+			endState: endState,
+			modeTransitions: [new ModeTransition(offsetDelta, this._languageIdentifier.sid)],
+		};
+	}
+
+	private _toBinaryTokens(tokens: modes.IToken2[], offsetDelta: number): Uint32Array {
+		let languageId = this._languageIdentifier.iid;
+		let theme = this._standaloneColorService.getTheme();
+
+		let result = new Uint32Array(tokens.length << 1);
+		for (let i = 0, len = tokens.length; i < len; i++) {
+			let t = tokens[i];
+			result[(i << 1)] = t.startIndex;
+			result[(i << 1) + 1] = theme.match(languageId, t.scopes);
+		}
+		return result;
+	}
+
+	public tokenize3(line: string, state: modes.IState, offsetDelta: number): modes.ILineTokens3 {
+		let actualResult = this._actual.tokenize(line, state);
+		let tokens = this._toBinaryTokens(actualResult.tokens, offsetDelta);
+
+		let endState: modes.IState;
+		// try to save an object if possible
+		if (actualResult.endState.equals(state)) {
+			endState = state;
+		} else {
+			endState = actualResult.endState;
+		}
+
+		return {
+			tokens: tokens,
+			endState: endState
+		};
+	}
 }
 
 /**
  * Set the tokens provider for a language (manual implementation).
  */
 export function setTokensProvider(languageId: string, provider: modes.TokensProvider): IDisposable {
-	let adapter = new TokenizationSupport2Adapter(languageId, provider);
+	let languageIdentifier = StaticServices.modeService.get().getLanguageIdentifier(languageId);
+	if (!languageIdentifier) {
+		throw new Error(`Cannot set tokens provider for unknown language ${languageId}`);
+	}
+	let adapter = new TokenizationSupport2Adapter(StaticServices.standaloneColorService.get(), languageIdentifier, provider);
 	return modes.TokenizationRegistry.register(languageId, adapter);
 }
 
@@ -77,7 +167,7 @@ export function setTokensProvider(languageId: string, provider: modes.TokensProv
  */
 export function setMonarchTokensProvider(languageId: string, languageDef: IMonarchLanguage): IDisposable {
 	let lexer = compile(languageId, languageDef);
-	let adapter = createTokenizationSupport(StaticServices.modeService.get(), languageId, lexer);
+	let adapter = createTokenizationSupport(StaticServices.modeService.get(), StaticServices.standaloneColorService.get(), languageId, lexer);
 	return modes.TokenizationRegistry.register(languageId, adapter);
 }
 
@@ -515,6 +605,9 @@ export function createMonacoLanguagesAPI(): typeof monaco.languages {
 		DocumentHighlightKind: modes.DocumentHighlightKind,
 		CompletionItemKind: CompletionItemKind,
 		SymbolKind: modes.SymbolKind,
-		IndentAction: IndentAction
+		IndentAction: IndentAction,
+
+		// classes
+		LanguageIdentifier: modes.LanguageIdentifier
 	};
 }

src/vs/editor/browser/standalone/standaloneServices.ts

@@ -39,6 +39,8 @@ import {
 import { ContextKeyService } from 'vs/platform/contextkey/browser/contextKeyService';
 import { IMenuService } from 'vs/platform/actions/common/actions';
 import { MenuService } from 'vs/platform/actions/common/menuService';
+import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
+import { StandaloneColorServiceImpl } from 'vs/editor/browser/services/standaloneColorServiceImpl';
 
 export interface IEditorContextViewService extends IContextViewService {
 	dispose(): void;
@@ -138,6 +140,8 @@ export module StaticServices {
 	export const progressService = define(IProgressService, () => new SimpleProgressService());
 
 	export const storageService = define(IStorageService, () => NullStorageService);
+
+	export const standaloneColorService = define(IStandaloneColorService, () => new StandaloneColorServiceImpl());
 }
 
 export class DynamicStandaloneServices extends Disposable {

src/vs/editor/browser/viewParts/lines/viewLines.css

@@ -16,8 +16,6 @@
 
 .monaco-editor .lines-content,
 .monaco-editor .view-line,
-.monaco-editor .view-line > span,
-.monaco-editor .view-line > span > span,
 .monaco-editor .view-lines {
 	-webkit-user-select: text;
 	-ms-user-select: text;
@@ -29,8 +27,6 @@
 
 .monaco-editor.ie .lines-content,
 .monaco-editor.ie .view-line,
-.monaco-editor.ie .view-line > span,
-.monaco-editor.ie .view-line > span > span,
 .monaco-editor.ie .view-lines {
 	-ms-user-select: none;
 	user-select: none;
@@ -56,9 +52,9 @@
 	top: 0;
 }
 
-/* bootstrap fix */
-.monaco-editor .view-line > span > span {
+/* TODO@tokenization bootstrap fix */
+/*.monaco-editor .view-line > span > span {
 	float: none;
 	min-height: inherit;
 	margin-left: inherit;
-}
+}*/

src/vs/editor/browser/widget/media/tokens.css

@@ -9,30 +9,31 @@
 .monaco-editor.hc-black .token {
 	mix-blend-mode: difference;
 }
-
-.monaco-editor.vs .token					{ color: #000000; }
 .monaco-editor.vs .token.vs-whitespace		{ color: rgba(51, 51, 51, 0.2) !important; }
+.monaco-editor.vs-dark .token.vs-whitespace	{ color: rgba(227, 228, 226, 0.16) !important; }
+.monaco-editor.hc-black .token.vs-whitespace{ color: rgba(227, 228, 226, 0.16) !important; }
+
+/* TODO@tokenization */
+/*.monaco-editor.vs .token					{ color: #000000; }
 .monaco-editor.vs .token.info-token			{ color: #316bcd; }
 .monaco-editor.vs .token.warn-token			{ color: #cd9731; }
 .monaco-editor.vs .token.error-token		{ color: #cd3131; }
 .monaco-editor.vs .token.debug-token		{ color: purple; }
 
 .monaco-editor.vs-dark .token				{ color: #D4D4D4; }
-.monaco-editor.vs-dark .token.vs-whitespace	{ color: rgba(227, 228, 226, 0.16) !important; }
 .monaco-editor.vs-dark .token.info-token	{ color: #6796e6; }
 .monaco-editor.vs-dark .token.warn-token	{ color: #cd9731; }
 .monaco-editor.vs-dark .token.error-token	{ color: #f44747; }
 .monaco-editor.vs-dark .token.debug-token	{ color: #b267e6; }
 
 .monaco-editor.hc-black .token				{ color: #FFFFFF; }
-.monaco-editor.hc-black .token.vs-whitespace{ color: rgba(227, 228, 226, 0.16) !important; }
 .monaco-editor.hc-black .token.info-token	{ color: #6796e6; }
 .monaco-editor.hc-black .token.warn-token	{ color: #008000; }
 .monaco-editor.hc-black .token.error-token	{ color: #FF0000; }
-.monaco-editor.hc-black .token.debug-token	{ color: #b267e6; }
+.monaco-editor.hc-black .token.debug-token	{ color: #b267e6; }*/
 
 
 /* ----- mimic text bundle support for themes ----- */
-.monaco-editor .markup.bold { font-weight: bold; }
+/*.monaco-editor .markup.bold { font-weight: bold; }
 .monaco-editor .markup.italic { font-style: italic; }
-.monaco-editor .markup.underline { text-decoration: underline; }
+.monaco-editor .markup.underline { text-decoration: underline; }*/

src/vs/editor/common/controller/cursorCollection.ts

@@ -330,7 +330,7 @@ export class CursorCollection {
 
 		let electricChars: string[] = null;
 		try {
-			electricChars = LanguageConfigurationRegistry.getElectricCharacters(this.model.getMode().getId());
+			electricChars = LanguageConfigurationRegistry.getElectricCharacters(this.model.getLanguageIdentifier().iid);
 		} catch (e) {
 			onUnexpectedError(e);
 			electricChars = null;
@@ -343,7 +343,7 @@ export class CursorCollection {
 
 		let autoClosingPairs: IAutoClosingPair[];
 		try {
-			autoClosingPairs = LanguageConfigurationRegistry.getAutoClosingPairs(this.model.getMode().getId());
+			autoClosingPairs = LanguageConfigurationRegistry.getAutoClosingPairs(this.model.getLanguageIdentifier().iid);
 		} catch (e) {
 			onUnexpectedError(e);
 			autoClosingPairs = null;
@@ -357,7 +357,7 @@ export class CursorCollection {
 
 		let surroundingPairs: IAutoClosingPair[];
 		try {
-			surroundingPairs = LanguageConfigurationRegistry.getSurroundingPairs(this.model.getMode().getId());
+			surroundingPairs = LanguageConfigurationRegistry.getSurroundingPairs(this.model.getLanguageIdentifier().iid);
 		} catch (e) {
 			onUnexpectedError(e);
 			surroundingPairs = null;

src/vs/editor/common/controller/textAreaState.ts

@@ -9,6 +9,7 @@ import { commonPrefixLength, commonSuffixLength } from 'vs/base/common/strings';
 import { Range } from 'vs/editor/common/core/range';
 import { EndOfLinePreference } from 'vs/editor/common/editorCommon';
 import { Position } from 'vs/editor/common/core/position';
+import { Constants } from 'vs/editor/common/core/uint';
 
 export interface IClipboardEvent {
 	canUseTextData(): boolean;
@@ -380,7 +381,7 @@ export class NVDAPagedTextAreaState extends TextAreaState {
 		let offset = page * NVDAPagedTextAreaState._LINES_PER_PAGE;
 		let startLineNumber = offset + 1;
 		let endLineNumber = offset + NVDAPagedTextAreaState._LINES_PER_PAGE;
-		return new Range(startLineNumber, 1, endLineNumber, Number.MAX_VALUE);
+		return new Range(startLineNumber, 1, endLineNumber, Constants.MAX_SAFE_SMALL_INTEGER);
 	}
 
 	public fromEditorSelection(model: ISimpleModel, selection: Range): TextAreaState {