Tokenization without extension contribution (closes #15)

src/main.ts

@@ -8,7 +8,7 @@ import { ConfigData } from './config/config';
 import { loadJsonFile } from './util/loader';
 import { getOniguruma } from './util/oniguruma';
 import { TextmateScopeSelector, TextmateScopeSelectorMap } from './util/selectors';
-import { ResolverService } from './services/resolver';
+import { isGrammarLanguageContribution, contributionKeys, ResolverService } from './services/resolver';
 import { OutlineService } from './services/outline';
 import { DocumentService } from './services/document';
 import { TextmateFoldingRangeProvider } from './folding';
@@ -17,14 +17,13 @@ import { TextmateDocumentSymbolProvider } from './document-symbol';
 import { TextmateWorkspaceSymbolProvider } from './workspace-symbol';
 
 import type { ConfigJson } from './config/config';
-import type { ExtensionManifest, GrammarLanguageContribution } from './services/resolver';
+import type { ExtensionManifest, GrammarContribution, LanguageContribution } from './services/resolver';
 
 export default class LSP {
 	public static utils = { TextmateScopeSelector, TextmateScopeSelectorMap, loadJsonFile };
 
 	// In order to support default class export we need to use `#` private properties.
 	// Refs: microsoft/TypeScript#30355
-	#_extensionManifest?: ExtensionManifest;
 	#_resolver: ResolverService;
 	#_registry: vscodeTextmate.Registry;
 	#_configPromise: Promise<ConfigData>;
@@ -38,23 +37,36 @@ export default class LSP {
 	#_definitionProvider?: TextmateDefinitionProvider;
 
 	constructor(public readonly languageId: string, public readonly context: vscode.ExtensionContext) {
-		this.#_extensionManifest = this.context.extension.packageJSON as ExtensionManifest;
+		const manifest = context.extension.packageJSON as ExtensionManifest;
+
+		const grammars: GrammarContribution[] = [];
+		const languages: LanguageContribution[] = [];
+		for (const key of contributionKeys) {
+			if (key in manifest === false) {
+				continue;
+			}
+			const contributes = manifest[key];
+			if (contributes.grammars && contributes.grammars.length) {
+				grammars.push(...contributes.grammars.filter(isGrammarLanguageContribution));
+			}
+			if (contributes.languages && contributes.languages.length) {
+				languages.push(...contributes.languages);
+			}
+		}
 
-		const contributes = this.#_extensionManifest?.contributes || {};
-		const grammars = (contributes?.grammars || [])
-			.filter((g): g is GrammarLanguageContribution => g && !g.injectTo);
-		const languages = contributes?.languages || [];
 		const onigLibPromise = getOniguruma();
-
 		this.#_resolver = new ResolverService(context, grammars, languages, onigLibPromise);
+
 		this.#_registry = new vscodeTextmate.Registry(this.#_resolver);
-		const grammarData = this.#_resolver.findGrammarByLanguageId(this.languageId);
+
+		const grammarData = this.#_resolver.findGrammarByLanguageId(languageId);
 		this.#_grammarPromise = this.#_registry.loadGrammar(grammarData.scopeName);
 
-		const mapping = this.#_extensionManifest['textmate-languageservices'] || {};
-		const path = mapping[this.languageId] || './textmate-configuration.json';
-		const uri = vscode.Uri.joinPath(this.context.extensionUri, path);
-		const languageData = this.#_resolver.findLanguageById(this.languageId);
+		const paths = manifest['textmate-languageservices'] || {};
+		const path = paths[languageId] || './textmate-configuration.json';
+
+		const uri = vscode.Uri.joinPath(context.extensionUri, path);
+		const languageData = this.#_resolver.findLanguageById(languageId);
 		this.#_configPromise = loadJsonFile<ConfigJson>(uri).then(json => new ConfigData(json, languageData));
 	}
 

src/services/resolver.ts

@@ -28,22 +28,47 @@ export interface GrammarInjectionContribution extends PartialJsonObject {
 
 export type GrammarContribution = GrammarLanguageContribution | GrammarInjectionContribution;
 
+export function isGrammarLanguageContribution(g: GrammarContribution): g is GrammarLanguageContribution {
+	return g && 'injectTo' in g === false;
+}
+
 export interface LanguageContribution extends PartialJsonObject {
 	id: string;
 	extensions?: string[];
 	filenames?: string[];
 }
 
+export interface ExtensionContributions extends PartialJsonObject {
+	grammars?: GrammarContribution[] & JsonArray;
+	languages?: LanguageContribution[] & JsonArray;
+}
+
+export interface LanguageConfigurations {
+	[languageId: string]: string;
+}
+
 export interface ExtensionManifest extends PackageJson {
-	contributes?: {
-		grammars?: GrammarContribution[] & JsonArray;
-		languages?: LanguageContribution[] & JsonArray;
-	};
-	'textmate-languageservices'?: { [languageId: string]: string };
+	contributes?: ExtensionContributions;
+	/** Mapping from language ID to config path. Default: `./textmate-configuration.json`. */
+	'textmate-languageservices'?: LanguageConfigurations;
+	/** Ersatz extension contributions - a service wiring to any language grammars. */
+	'textmate-languageservice-contributes'?: ExtensionContributions;
 }
 
+export const contributionKeys: ExtensionManifestContributionKey[] = [
+	'contributes',
+	'textmate-languageservice-contributes'
+];
+
+export type ExtensionManifestContributionKey = 'contributes' | 'textmate-languageservice-contributes';
+
 export class ResolverService implements vscodeTextmate.RegistryOptions {
-	constructor(private _context: vscode.ExtensionContext, private _grammars: GrammarContribution[], private _languages: LanguageContribution[], public onigLib: Promise<vscodeTextmate.IOnigLib>) {
+	constructor(
+		private _context: vscode.ExtensionContext,
+		private _grammars: GrammarContribution[],
+		private _languages: LanguageContribution[],
+		public onigLib: Promise<vscodeTextmate.IOnigLib>
+	) {
 	}
 
 	public findLanguageByExtension(fileExtension: string): string | null {
@@ -92,7 +117,7 @@ export class ResolverService implements vscodeTextmate.RegistryOptions {
 				return language;
 			}
 		}
-		throw new Error('Could not find language contribution for language ID "' + id + '"');
+		throw new Error('Could not find language contribution for language ID "' + id + '" in extension manifest');
 	}
 
 	public findGrammarByLanguageId(id: string): GrammarContribution {
@@ -101,7 +126,7 @@ export class ResolverService implements vscodeTextmate.RegistryOptions {
 				return grammar;
 			}
 		}
-		throw new Error('Could not find grammar contribution for language ID "' + id + '"');
+		throw new Error('Could not find grammar contribution for language ID "' + id + '" in extension manifest');
 	}
 
 	public async loadGrammar(scopeName: string): Promise<vscodeTextmate.IRawGrammar | null> {

src/util/oniguruma.ts

@@ -9,22 +9,24 @@ import type * as vscodeTextmate from 'vscode-textmate';
 
 // Use webpack + encoded-uint8array-loader to generate a `Uint8Array` WASM module.
 // This is not streaming :[ but vscode libs must bundle WASM deps to support web ecosystem.
-// Better alternative is using copy-webpack-plugin + fetch to include the WASM file.
+// Extension alternative is using copy-webpack-plugin + fetch to include the WASM file.
+// TODO: use data URI and native node 18.x fetch for streaming compilation.
 import * as data from '../../node_modules/vscode-oniguruma/release/onig.wasm';
 
 let onigurumaLib: vscodeTextmate.IOnigLib | null = null;
 
 export async function getOniguruma(): Promise<vscodeTextmate.IOnigLib> {
-	if (!onigurumaLib) {
-		await vscodeOniguruma.loadWASM({ data });
-		onigurumaLib = {
-			createOnigScanner(patterns: string[]) {
-				return new vscodeOniguruma.OnigScanner(patterns);
-			},
-			createOnigString(str: string) {
-				return new vscodeOniguruma.OnigString(str);
-			}
-		};
+	if (onigurumaLib) {
+		return onigurumaLib;
 	}
+	await vscodeOniguruma.loadWASM({ data });
+	onigurumaLib = {
+		createOnigScanner(patterns: string[]) {
+			return new vscodeOniguruma.OnigScanner(patterns);
+		},
+		createOnigString(str: string) {
+			return new vscodeOniguruma.OnigString(str);
+		}
+	};
 	return onigurumaLib;
 }

test/util/jsonify.ts

@@ -19,11 +19,7 @@ function replaceClassesWithStrings(key: string, value: any): any {
 
 	// Internal {@link vscode.Uri} class has the constructor type Object.
 	// Sometimes numerous fields are missing also.
-	if (
-		['', 'uri'].includes(key) &&
-		!!value && typeof value === 'object' &&
-		Object.prototype.hasOwnProperty.call(value, 'path')
-	) {
+	if (['', 'uri'].includes(key) && !!value && typeof value === 'object' && 'path' in value) {
 		const externalPath = getNormalizedPathFor(value as vscode.Uri);
 		const extensionPath = getNormalizedPathFor(extensionContext.extensionUri);
 		return './' + path.posix.relative(extensionPath, externalPath);