fix: Update grammar app (#3070)

streetsidesoftware · Jun 15, 2022 · 9e0860f · 9e0860f
1 parent 0361d5f
commit 9e0860f
Show file tree

Hide file tree

Showing 10 changed files with 142 additions and 31 deletions.
diff --git a/packages/cspell-grammar/fixtures/TypeScript/sample1.ts b/packages/cspell-grammar/fixtures/TypeScript/sample1.ts
@@ -0,0 +1,8 @@
+/**
+ * This is some sample code
+ */
+
+export const regExpPath = /(\w+\/)+/g;
+export const regExpPath2 = /(\w+[/\\])+/g;
+
+export const comment = 'This is a comment string';
diff --git a/packages/cspell-grammar/package.json b/packages/cspell-grammar/package.json
@@ -1,7 +1,6 @@
 {
   "name": "cspell-grammar",
   "version": "6.1.2",
-  "private": true,
   "description": "Grammar parsing support for cspell",
   "keywords": [
     "cspell",

diff --git a/packages/cspell-grammar/src/__snapshots__/app.test.ts.snap b/packages/cspell-grammar/src/__snapshots__/app.test.ts.snap
@@ -0,0 +1,25 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`app app TypeScript/sample1.ts 1`] = `
+"File: sample1.ts Parser: typescript
+3-4	↩︎	comment.block.documentation.ts code.ts source.ts
+4-32	 * This is some sample code↩︎	comment.block.documentation.ts code.ts source.ts
+32-33	 	comment.block.documentation.ts code.ts source.ts
+50-60	regExpPath	identifier code.ts source.ts
+64-65	(	regexp.ts code.ts source.ts
+65-67	\\\\w	escape.regexp.ts regexp.ts code.ts source.ts
+67-68	+	regexp.ts code.ts source.ts
+68-70	\\\\/	escape.regexp.ts regexp.ts code.ts source.ts
+70-72	)+	regexp.ts code.ts source.ts
+89-100	regExpPath2	identifier code.ts source.ts
+104-105	(	regexp.ts code.ts source.ts
+105-107	\\\\w	escape.regexp.ts regexp.ts code.ts source.ts
+107-108	+	regexp.ts code.ts source.ts
+108-109	[	brace.regexp.ts regexp.ts code.ts source.ts
+109-110	/	character-class.regexp.ts brace.regexp.ts regexp.ts code.ts source.ts
+110-112	\\\\\\\\	escape.regexp.ts character-class.regexp.ts brace.regexp.ts regexp.ts code.ts source.ts
+112-113	]	brace.regexp.ts regexp.ts code.ts source.ts
+113-115	)+	regexp.ts code.ts source.ts
+133-140	comment	identifier code.ts source.ts
+144-168	This is a comment string	string.quoted.single.ts code.ts source.ts"
+`;
diff --git a/packages/cspell-grammar/src/app.test.ts b/packages/cspell-grammar/src/app.test.ts
@@ -0,0 +1,18 @@
+import * as path from 'path';
+import { run } from './app';
+
+describe('app', () => {
+    test.each`
+        filename
+        ${'TypeScript/sample1.ts'}
+    `('app $filename', async ({ filename }) => {
+        const log = jest.spyOn(console, 'log').mockImplementation();
+        await run(['', '', r(filename)]);
+        expect(log.mock.calls.map((c) => c.join(';')).join('\n')).toMatchSnapshot();
+        log.mockRestore();
+    });
+});
+
+function r(file: string): string {
+    return path.resolve(path.join(__dirname, '..', 'fixtures'), file);
+}
diff --git a/packages/cspell-grammar/src/app.ts b/packages/cspell-grammar/src/app.ts
@@ -1,12 +1,10 @@
-import { TypeScript } from './grammars';
-import { NGrammar } from './parser/grammarNormalized';
-import { normalizeGrammar } from './parser/grammarNormalizer';
-import * as path from 'path';
+import type { ParsedText, Parser } from '@cspell/cspell-types/Parser';
 import { promises as fs } from 'fs';
-import { parseDocument } from './parser/parser';
+import * as path from 'path';
+import { parser as parserTypeScript } from './parsers/typescript';
 
-const grammars: Record<string, NGrammar | undefined> = {
-    '.ts': normalizeGrammar(TypeScript.grammar),
+const parsers: Record<string, Parser> = {
+    '.ts': parserTypeScript,
 };
 
 /**
@@ -15,7 +13,6 @@ const grammars: Record<string, NGrammar | undefined> = {
  * @returns Promise
  */
 export async function run(args: string[]): Promise<void> {
-    console.log('args: %o', args);
     // early out if there are not enough arguments
     if (args.length < 3) {
         console.log('usage...');
@@ -29,14 +26,22 @@ export async function run(args: string[]): Promise<void> {
     }
 
     const ext = path.extname(filename);
-    const g = grammars[ext];
-    if (!g) {
-        console.log(`No grammar for ${path.basename(filename)}`);
+    const parser = parsers[ext];
+    if (!parser) {
+        console.log(`No parser for ${path.basename(filename)}`);
         return;
     }
 
-    console.log(`File: ${path.basename(filename)} Grammar: ${g.name || g.scopeName}`);
+    console.log(`File: ${path.basename(filename)} Parser: ${parser.name}`);
     const content = await fs.readFile(filename, 'utf-8');
 
-    parseDocument(g, filename, content);
+    const result = parser.parse(content, filename);
+    for (const pt of result.parsedTexts) {
+        emit(pt);
+    }
+}
+
+function emit(pt: ParsedText) {
+    const t = pt.text.replace(/\t/g, '↦').replace(/\r?\n/g, '↩︎').replace(/\r/g, '⇠');
+    console.log(`${pt.range[0]}-${pt.range[1]}\t${t}\t${pt.scope?.toString() || ''}`);
 }
diff --git a/packages/cspell-grammar/src/grammars/typescript.ts b/packages/cspell-grammar/src/grammars/typescript.ts
@@ -5,6 +5,7 @@ const repository: Repository = {
         name: 'code.ts',
         patterns: [
             '#keyword',
+            '#regexp',
             '#string',
             '#comment',
             '#braces',
@@ -126,6 +127,25 @@ const repository: Repository = {
             },
         ],
     },
+    regexp: {
+        name: 'regexp.ts',
+        begin: /\/(?![/*])/,
+        end: /\/([a-z]*)/i,
+        beginCaptures: 'punctuation.begin.regexp.ts',
+        endCaptures: 'punctuation.end.regexp.ts',
+        patterns: ['#regexp_escape', '#regexp_brace'],
+    },
+    regexp_escape: {
+        name: 'escape.regexp.ts',
+        match: /\\./,
+    },
+    regexp_brace: {
+        name: 'brace.regexp.ts',
+        begin: '[',
+        end: ']',
+        contentName: 'character-class.regexp.ts',
+        patterns: ['#regexp_escape'],
+    },
 };
 
 export const grammar: GrammarDef = {

diff --git a/packages/cspell-grammar/src/parser/processors/procMatchingRule.ts b/packages/cspell-grammar/src/parser/processors/procMatchingRule.ts
@@ -30,7 +30,7 @@ export function applyCaptureToEnd(rule: Rule, match: MatchResult): TokenizedText
 
     const bePattern = <NPatternBeginEnd>pattern;
 
-    const captures = bePattern.beginCaptures ?? bePattern.captures;
+    const captures = bePattern.endCaptures ?? bePattern.captures;
 
     return applyCaptures(rule, match, captures);
 }

diff --git a/packages/cspell-grammar/src/parsers/typescript/TypeScriptParser.ts b/packages/cspell-grammar/src/parsers/typescript/TypeScriptParser.ts
@@ -1,17 +1,19 @@
-import { grammar } from '../../grammars/typescript';
-import { compileGrammar } from '../..';
-import { createParser } from '../../parser/parser';
-import { TokenizedLine } from '../../parser/types';
-import { ParsedText, ParseResult, Scope } from '@cspell/cspell-types/Parser';
 import { opFlatten, opMap, pipeSync as pipe } from '@cspell/cspell-pipe';
+import { ParsedText, ParseResult, Scope, ScopeChain } from '@cspell/cspell-types/Parser';
+import { compileGrammar } from '../..';
+import { grammar } from '../../grammars/typescript';
+import { appendMappedText } from '../../mappers/appendMappedText';
 import { mapRawString } from '../../mappers/typescript';
+import { createParser } from '../../parser/parser';
 import { ScopePool } from '../../parser/scope';
-import { appendMappedText } from '../../mappers/appendMappedText';
+import { TokenizedLine } from '../../parser/types';
 
 const tsGrammar = compileGrammar(grammar);
 
 const pool = new ScopePool();
 
+const useScope = new WeakMap<ScopeChain, boolean>();
+
 function* transform(texts: ParseResult['parsedTexts']): ParseResult['parsedTexts'] {
     for (const parsed of texts) {
         if (doesScopeMatch(parsed.scope, 'constant.character.escape.ts')) {
@@ -67,9 +69,18 @@ function mergeParsedText(a: ParsedText, b: ParsedText): ParsedText {
     return ab;
 }
 
+function filterScope(scope: ScopeChain): boolean {
+    const cached = useScope.get(scope);
+    if (cached !== undefined) return cached;
+    const value = scope.value;
+    const use = !value.startsWith('punctuation') && !value.startsWith('keyword.');
+    useScope.set(scope, use);
+    return use;
+}
+
 function mapTokenizedLine(tl: TokenizedLine): ParseResult['parsedTexts'] {
     return tl.tokens
-        .filter((t) => !t.scope.value.startsWith('punctuation'))
+        .filter((t) => filterScope(t.scope))
         .map((t) => ({
             text: t.text,
             range: [tl.offset + t.range[0], tl.offset + t.range[1]] as const,

diff --git a/packages/cspell-grammar/src/parsers/typescript/__snapshots__/index.test.ts.snap b/packages/cspell-grammar/src/parsers/typescript/__snapshots__/index.test.ts.snap
@@ -1,26 +1,48 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`TypeScript Parser parse sample 1`] = `
+exports[`TypeScript Parser parse TypeScript/escape-codes.ts 1`] = `
 "filename: TypeScript/escape-codes.ts
 parsed:
-0-5 \\"const\\" keyword.typescript.ts code.ts source.ts
 6-12 \\"places\\" identifier code.ts source.ts
 17-26 \\"Amsterdam\\" string.quoted.single.ts code.ts code.ts meta.brace.ts code.ts source.ts
 30-40 \\"Orléans\\" string.quoted.single.ts code.ts code.ts meta.brace.ts code.ts source.ts
-45-50 \\"const\\" keyword.typescript.ts code.ts source.ts
 51-57 \\"sample\\" identifier code.ts source.ts
 66-70 \\"text\\" identifier code.ts code.ts meta.brace.ts code.ts source.ts
 73-162 \\"Fingerspitzengefühl is a German term.\\\\nIt’s pronounced as follows: [ˈfɪŋɐˌʃpɪtsənɡəˌfyːl]\\" string.quoted.single.ts code.ts code.ts meta.brace.ts code.ts source.ts
 169-172 \\"hex\\" identifier code.ts code.ts meta.brace.ts code.ts source.ts
 175-557 \\"Fingerspitzengefühl is a German term.\\\\nIt’s pronounced as follows: [ˈfɪŋɐˌʃpɪtsənɡəˌfyːl]\\" string.quoted.single.ts code.ts code.ts meta.brace.ts code.ts source.ts
 564-569 \\"mixed\\" identifier code.ts code.ts meta.brace.ts code.ts source.ts
 572-731 \\"Fingerspitzengefühl is a German term.\\\\nIt’s pronounced as follows: [ˈfɪŋɐˌʃpɪtsənɡəˌfyːl]\\" string.quoted.single.ts code.ts code.ts meta.brace.ts code.ts source.ts
-738-745 \\"console\\" keyword.lib.ts code.ts source.ts
 745-749 \\".log\\" identifier code.ts source.ts
 750-756 \\"places\\" identifier code.ts code.ts meta.brace.ts code.ts source.ts
-759-766 \\"console\\" keyword.lib.ts code.ts source.ts
 766-770 \\".log\\" identifier code.ts source.ts
 772-774 \\"%o\\" string.quoted.single.ts code.ts code.ts meta.brace.ts code.ts source.ts
 777-783 \\"sample\\" identifier code.ts code.ts meta.brace.ts code.ts source.ts
 "
 `;
+
+exports[`TypeScript Parser parse TypeScript/sample1.ts 1`] = `
+"filename: TypeScript/sample1.ts
+parsed:
+3-4 \\"\\\\n\\" comment.block.documentation.ts code.ts source.ts
+4-32 \\" * This is some sample code\\\\n\\" comment.block.documentation.ts code.ts source.ts
+32-33 \\" \\" comment.block.documentation.ts code.ts source.ts
+50-60 \\"regExpPath\\" identifier code.ts source.ts
+64-65 \\"(\\" regexp.ts code.ts source.ts
+65-67 \\"\\\\\\\\w\\" escape.regexp.ts regexp.ts code.ts source.ts
+67-68 \\"+\\" regexp.ts code.ts source.ts
+68-70 \\"\\\\\\\\/\\" escape.regexp.ts regexp.ts code.ts source.ts
+70-72 \\")+\\" regexp.ts code.ts source.ts
+89-100 \\"regExpPath2\\" identifier code.ts source.ts
+104-105 \\"(\\" regexp.ts code.ts source.ts
+105-107 \\"\\\\\\\\w\\" escape.regexp.ts regexp.ts code.ts source.ts
+107-108 \\"+\\" regexp.ts code.ts source.ts
+108-109 \\"[\\" brace.regexp.ts regexp.ts code.ts source.ts
+109-110 \\"/\\" character-class.regexp.ts brace.regexp.ts regexp.ts code.ts source.ts
+110-112 \\"\\\\\\\\\\\\\\\\\\" escape.regexp.ts character-class.regexp.ts brace.regexp.ts regexp.ts code.ts source.ts
+112-113 \\"]\\" brace.regexp.ts regexp.ts code.ts source.ts
+113-115 \\")+\\" regexp.ts code.ts source.ts
+133-140 \\"comment\\" identifier code.ts source.ts
+144-168 \\"This is a comment string\\" string.quoted.single.ts code.ts source.ts
+"
+`;
diff --git a/packages/cspell-grammar/src/parsers/typescript/index.test.ts b/packages/cspell-grammar/src/parsers/typescript/index.test.ts
@@ -6,14 +6,17 @@ import { ParseResult } from '@cspell/cspell-types/Parser';
 const fixtures = path.join(__dirname, '../../../fixtures');
 
 describe('TypeScript Parser', () => {
-    const sample1File = 'TypeScript/escape-codes.ts';
     test('parser', () => {
         expect(parser.name).toBe('typescript');
     });
 
-    test('parse sample', async () => {
-        const content = await readSample(sample1File);
-        const p = parser.parse(content, sample1File);
+    test.each`
+        filename
+        ${'TypeScript/escape-codes.ts'}
+        ${'TypeScript/sample1.ts'}
+    `('parse $filename', async ({ filename }) => {
+        const content = await readSample(filename);
+        const p = parser.parse(content, filename);
         expect(stringifyResult(p)).toMatchSnapshot();
     });
 });