feat: add --cache option to lint only changed files (#1763)

* add --cache option * update help * add jsdoc * add --cache-strategy * do not try to lint directories * hash config for every file * fix ts error * add test-bin scripts * Update cSpell.json * Ensure that `meta` is defined. Co-authored-by: Jason Dent <[email protected]>
streetsidesoftware · Sep 27, 2021 · 4bdfd09 · 4bdfd09
1 parent c0aaf45
commit 4bdfd09
Show file tree

Hide file tree

Showing 24 changed files with 556 additions and 137 deletions.
diff --git a/.gitignore b/.gitignore
@@ -97,4 +97,7 @@ typings/
 
 .tsbuildinfo
 
-# cspell:ignore pids jscoverage wscript jspm dotenv eslintcache
+# cspell cache
+.cspellcache
+
+# cspell:ignore pids jscoverage wscript jspm dotenv eslintcache cspellcache
diff --git a/package.json b/package.json
@@ -36,9 +36,11 @@
     "symlink": "node build_tools/bin/symlink.js packages test-packages",
     "test-watch": "lerna run --parallel test-watch",
     "test": "lerna run test && npm run test-schema",
-    "test-bin": "npm run test-bin-spell && npm run test-bin-trace && npm run test-bin-check",
+    "test-bin": "npm run test-bin-spell && npm run test-bin-spell-cache-content && npm run test-bin-spell-cache-metadata && npm run test-bin-trace && npm run test-bin-check",
     "test-bin-check": "node ./bin.js check README.md",
     "test-bin-spell": "node ./bin.js -c cspellrc.json",
+    "test-bin-spell-cache-content": "node ./bin.js -c cspellrc.json --cache --cache-strategy content",
+    "test-bin-spell-cache-metadata": "node ./bin.js -c cspellrc.json --cache --cache-strategy metadata",
     "test-bin-trace": "node ./bin.js trace test",
     "test-integrations": "cd ./integration-tests && npm run integration-tests",
     "test-schema": "ajv -s ./cspell.schema.json -d cspell.json -c ./packages/cspell-types/ajv.config.js",

diff --git a/packages/cspell-json-reporter/src/__snapshots__/index.test.ts.snap b/packages/cspell-json-reporter/src/__snapshots__/index.test.ts.snap
@@ -12,7 +12,6 @@ exports[`getReporter saves additional data 1`] = `
             },
             \\"row\\": 1,
             \\"col\\": 14,
-            \\"doc\\": \\"This text is fulll of errrorrrs.\\",
             \\"uri\\": \\"text.txt\\",
             \\"context\\": {
                 \\"text\\": \\"This text is fulll of errrorrrs.\\",
@@ -82,7 +81,6 @@ exports[`getReporter saves json to file 1`] = `
             },
             \\"row\\": 1,
             \\"col\\": 14,
-            \\"doc\\": \\"This text is fulll of errrorrrs.\\",
             \\"uri\\": \\"text.txt\\",
             \\"context\\": {
                 \\"text\\": \\"This text is fulll of errrorrrs.\\",

diff --git a/packages/cspell-json-reporter/src/index.test.ts b/packages/cspell-json-reporter/src/index.test.ts
@@ -58,7 +58,6 @@ async function runReporter(reporter: CSpellReporter): Promise<void> {
         line: { text: 'This text is fulll of errrorrrs.', offset: 0 },
         row: 1,
         col: 14,
-        doc: 'This text is fulll of errrorrrs.',
         uri: 'text.txt',
         context: { text: 'This text is fulll of errrorrrs.', offset: 0 },
     });

diff --git a/packages/cspell-types/src/CSpellReporter.ts b/packages/cspell-types/src/CSpellReporter.ts
@@ -1,6 +1,6 @@
 import { TextDocumentOffset, TextOffset } from './TextOffset';
 
-export interface Issue extends TextDocumentOffset {
+export interface Issue extends Omit<TextDocumentOffset, 'doc'> {
     /** text surrounding the issue text */
     context: TextOffset;
     isFlagged?: boolean;

diff --git a/packages/cspell/package-lock.json b/packages/cspell/package-lock.json
diff --git a/packages/cspell/package.json b/packages/cspell/package.json
@@ -69,14 +69,19 @@
   },
   "homepage": "https://streetsidesoftware.github.io/cspell/",
   "dependencies": {
+    "@types/file-entry-cache": "^5.0.2",
+    "@types/imurmurhash": "^0.1.1",
     "chalk": "^4.1.2",
     "commander": "^8.2.0",
     "comment-json": "^4.1.1",
     "cspell-glob": "^5.10.0",
     "cspell-lib": "^5.10.1",
+    "fast-json-stable-stringify": "^2.1.0",
+    "file-entry-cache": "^6.0.1",
     "fs-extra": "^10.0.0",
     "get-stdin": "^8.0.0",
     "glob": "^7.2.0",
+    "imurmurhash": "^0.1.4",
     "strip-ansi": "^6.0.1",
     "vscode-uri": "^3.0.2"
   },

diff --git a/packages/cspell/src/__snapshots__/app.test.ts.snap b/packages/cspell/src/__snapshots__/app.test.ts.snap
@@ -269,38 +269,43 @@ Array [
   "Check spelling",
   "",
   "Options:",
-  "  -c, --config <cspell.json>  Configuration file to use.  By default cspell",
-  "                              looks for cspell.json in the current directory.",
-  "  -v, --verbose               display more information about the files being",
-  "                              checked and the configuration",
-  "  --locale <locale>           Set language locales. i.e. \\"en,fr\\" for English",
-  "                              and French, or \\"en-GB\\" for British English.",
-  "  --language-id <language>    Force programming language for unknown",
-  "                              extensions. i.e. \\"php\\" or \\"scala\\"",
-  "  --languageId <language>     Force programming language for unknown",
-  "                              extensions. i.e. \\"php\\" or \\"scala\\"",
-  "  --wordsOnly                 Only output the words not found in the",
-  "                              dictionaries.",
-  "  -u, --unique                Only output the first instance of a word not",
-  "                              found in the dictionaries.",
-  "  --debug                     Output information useful for debugging",
-  "                              cspell.json files.",
-  "  -e, --exclude <glob>        Exclude files matching the glob pattern. This",
-  "                              option can be used multiple times to add multiple",
-  "                              globs.",
-  "  --no-issues                 Do not show the spelling errors.",
-  "  --no-progress               Turn off progress messages",
-  "  --no-summary                Turn off summary message in console",
-  "  -s, --silent                Silent mode, suppress error messages",
-  "  -r, --root <root folder>    Root directory, defaults to current directory.",
-  "  --relative                  Issues are displayed relative to root.",
-  "  --show-context              Show the surrounding text around an issue.",
-  "  --show-suggestions          Show spelling suggestions.",
-  "  --must-find-files           Error if no files are found (default: true)",
-  "  --no-must-find-files        Do not error if no files are found",
-  "  --legacy                    Legacy output",
-  "  --local <local>             Deprecated -- Use: --locale",
-  "  -h, --help                  display help for command",
+  "  -c, --config <cspell.json>   Configuration file to use.  By default cspell",
+  "                               looks for cspell.json in the current directory.",
+  "  -v, --verbose                display more information about the files being",
+  "                               checked and the configuration",
+  "  --locale <locale>            Set language locales. i.e. \\"en,fr\\" for English",
+  "                               and French, or \\"en-GB\\" for British English.",
+  "  --language-id <language>     Force programming language for unknown",
+  "                               extensions. i.e. \\"php\\" or \\"scala\\"",
+  "  --languageId <language>      Force programming language for unknown",
+  "                               extensions. i.e. \\"php\\" or \\"scala\\"",
+  "  --wordsOnly                  Only output the words not found in the",
+  "                               dictionaries.",
+  "  -u, --unique                 Only output the first instance of a word not",
+  "                               found in the dictionaries.",
+  "  --debug                      Output information useful for debugging",
+  "                               cspell.json files.",
+  "  -e, --exclude <glob>         Exclude files matching the glob pattern. This",
+  "                               option can be used multiple times to add",
+  "                               multiple globs.",
+  "  --no-issues                  Do not show the spelling errors.",
+  "  --no-progress                Turn off progress messages",
+  "  --no-summary                 Turn off summary message in console",
+  "  -s, --silent                 Silent mode, suppress error messages",
+  "  -r, --root <root folder>     Root directory, defaults to current directory.",
+  "  --relative                   Issues are displayed relative to root.",
+  "  --show-context               Show the surrounding text around an issue.",
+  "  --show-suggestions           Show spelling suggestions.",
+  "  --must-find-files            Error if no files are found (default: true)",
+  "  --no-must-find-files         Do not error if no files are found",
+  "  --legacy                     Legacy output",
+  "  --local <local>              Deprecated -- Use: --locale",
+  "  --cache                      Only check changed files (default: false)",
+  "  --cache-strategy <strategy>  Strategy to use for detecting changed files",
+  "                               (choices: \\"metadata\\", \\"content\\")",
+  "  --cache-location <path>      Path to the cache file or directory (default:",
+  "                               \\".cspellcache\\")",
+  "  -h, --help                   display help for command",
   "",
   "",
   "Examples:",
@@ -547,16 +552,12 @@ exports[`Validate cli app trace not-in-any-dictionary Expect Error: [Function Ch
 exports[`Validate cli app with errors and excludes Expect Error: [Function CheckFailed] 1`] = `Array []`;
 
 exports[`Validate cli app with errors and excludes Expect Error: [Function CheckFailed] 2`] = `
-"error	 1/10 ./samples/comments_only -
-error	 2/10 ./samples/cspell-bad.json 0.00ms
-error	 3/10 ./samples/cspell-includes.json 0.00ms
-error	 4/10 ./samples/cspell-missing-dict.json 0.00ms
-error	 5/10 ./samples/latex -
-error	 6/10 ./samples/linked -
-error	 7/10 ./samples/sample.c 0.00ms
-error	 8/10 ./samples/sample.py 0.00ms
-error	 9/10 ./samples/src -
-error	10/10 ./samples/text.txt 0.00ms X
+"error	1/6 ./samples/cspell-bad.json 0.00ms
+error	2/6 ./samples/cspell-includes.json 0.00ms
+error	3/6 ./samples/cspell-missing-dict.json 0.00ms
+error	4/6 ./samples/sample.c 0.00ms
+error	5/6 ./samples/sample.py 0.00ms
+error	6/6 ./samples/text.txt 0.00ms X
 log	./samples/text.txt:1:14 - Unknown word (fulll)
 log	./samples/text.txt:1:23 - Unknown word (errrorrrs)
 log	./samples/text.txt:3:4 - Unknown word (hass)

diff --git a/packages/cspell/src/app.ts b/packages/cspell/src/app.ts
@@ -17,6 +17,7 @@ import { tableToLines } from './util/table';
 import { emitTraceResults } from './traceEmitter';
 import { getReporter } from './cli-reporter';
 import { CheckFailed } from './util/errors';
+import { DEFAULT_CACHE_LOCATION } from './util/cache';
 
 export { CheckFailed } from './util/errors';
 
@@ -84,6 +85,14 @@ export async function run(program?: commander.Command, argv?: string[]): Promise
         // .option('--force', 'Force the exit value to always be 0')
         .option('--legacy', 'Legacy output')
         .option('--local <local>', 'Deprecated -- Use: --locale')
+        .option('--cache', 'Only check changed files', false)
+        .addOption(
+            new commander.Option('--cache-strategy <strategy>', 'Strategy to use for detecting changed files').choices([
+                'metadata',
+                'content',
+            ])
+        )
+        .option('--cache-location <path>', `Path to the cache file or directory`, DEFAULT_CACHE_LOCATION)
         .addHelpText('after', usage)
         .arguments('[files...]')
         .action((files: string[], options: Options) => {

diff --git a/packages/cspell/src/cli-reporter.test.ts b/packages/cspell/src/cli-reporter.test.ts
@@ -50,7 +50,6 @@ function genIssue(word: string): ReporterIssue {
         offset,
         uri: 'file://uri/path/filename',
         context,
-        doc,
         line,
         col,
         row,

diff --git a/packages/cspell/src/fileHelper.ts b/packages/cspell/src/fileHelper.ts
@@ -3,7 +3,7 @@ import * as fsp from 'fs-extra';
 import getStdin from 'get-stdin';
 import { GlobOptions, globP } from './util/glob';
 import * as path from 'path';
-import { CSpellUserSettings, Document, fileToDocument } from 'cspell-lib';
+import { CSpellUserSettings, Document, fileToDocument, Issue } from 'cspell-lib';
 
 const UTF8: BufferEncoding = 'utf8';
 const STDIN = 'stdin';
@@ -31,7 +31,15 @@ export async function readConfig(configFile: string | undefined, root: string |
 
 export interface FileInfo {
     filename: string;
-    text: string;
+    text?: string;
+}
+export interface FileResult {
+    fileInfo: FileInfo;
+    processed: boolean;
+    issues: Issue[];
+    errors: number;
+    configErrors: number;
+    elapsedTimeMs: number;
 }
 
 export function fileInfoToDocument(
@@ -54,7 +62,7 @@ export function fileInfoToDocument(
     return fileToDocument(filename, text, languageId, locale);
 }
 
-export function readFileInfo(filename: string, encoding: string = UTF8): Promise<FileInfo> {
+export function readFileInfo(filename: string, encoding: string = UTF8): Promise<Required<FileInfo>> {
     const pText = filename === STDIN ? getStdin() : fsp.readFile(filename, encoding);
     return pText.then(
         (text) => ({ text, filename }),