diff --git a/packages/angular_devkit/build_angular/test/browser/rebuild_spec_large.ts b/packages/angular_devkit/build_angular/test/browser/rebuild_spec_large.ts
index 7ec084e7b15c..892bdadc3581 100644
--- a/packages/angular_devkit/build_angular/test/browser/rebuild_spec_large.ts
+++ b/packages/angular_devkit/build_angular/test/browser/rebuild_spec_large.ts
@@ -268,6 +268,41 @@ describe('Browser Builder rebuilds', () => {
       .toPromise();
   });
 
+  it('rebuilds on transitive non node package DTS file changes', async () => {
+    host.writeMultipleFiles({
+      'src/interface1.d.ts': `
+        import { Interface2 } from './interface2';
+        export interface Interface1 extends Interface2 { }
+      `,
+      'src/interface2.d.ts': `
+        import { Interface3 } from './interface3';
+        export interface Interface2 extends Interface3 { }
+      `,
+      'src/interface3.d.ts': `export interface Interface3 { nbr: number; }`,
+    });
+    host.appendToFile('src/main.ts', `
+      import { Interface1 } from './interface1';
+      const something: Interface1 = { nbr: 43 };
+    `);
+
+    const overrides = { watch: true };
+    const run = await architect.scheduleTarget(target, overrides);
+    let buildNumber = 0;
+    await run.output
+      .pipe(
+        debounceTime(rebuildDebounceTime),
+        tap(buildEvent => expect(buildEvent.success).toBe(true)),
+        tap(() => {
+          buildNumber++;
+          if (buildNumber === 1) {
+            host.appendToFile('src/interface3.d.ts', 'export declare type MyType = string;');
+          }
+        }),
+        take(2),
+      )
+      .toPromise();
+  });
+
   it('rebuilds after errors in JIT', async () => {
     const origContent = virtualFs.fileBufferToString(
       host.scopedSync().read(normalize('src/app/app.component.ts')),
diff --git a/packages/ngtools/webpack/src/angular_compiler_plugin.ts b/packages/ngtools/webpack/src/angular_compiler_plugin.ts
index d710281085c9..264bebbaeb6d 100644
--- a/packages/ngtools/webpack/src/angular_compiler_plugin.ts
+++ b/packages/ngtools/webpack/src/angular_compiler_plugin.ts
@@ -385,13 +385,44 @@ export class AngularCompilerPlugin {
     }
 
     const newTsProgram = this._getTsProgram();
-    if (oldTsProgram && newTsProgram) {
+    const newProgramSourceFiles = newTsProgram?.getSourceFiles();
+    const localDtsFiles = new Set(
+      newProgramSourceFiles?.filter(
+        f => f.isDeclarationFile && !this._nodeModulesRegExp.test(f.fileName),
+      )
+      .map(f => this._compilerHost.denormalizePath(f.fileName)),
+    );
+
+    if (!oldTsProgram) {
+      // Add all non node package dts files as depedencies when not having an old program
+      for (const dts of localDtsFiles) {
+        this._typeDeps.add(dts);
+      }
+    } else if (oldTsProgram && newProgramSourceFiles) {
       // The invalidation should only happen if we have an old program
       // as otherwise we will invalidate all the sourcefiles.
       const oldFiles = new Set(oldTsProgram.getSourceFiles().map(sf => sf.fileName));
-      const newFiles = newTsProgram.getSourceFiles().filter(sf => !oldFiles.has(sf.fileName));
-      for (const newFile of newFiles) {
-        this._compilerHost.invalidate(newFile.fileName);
+      const newProgramFiles = new Set(newProgramSourceFiles.map(sf => sf.fileName));
+
+      for (const dependency of this._typeDeps) {
+        // Remove type dependencies of no longer existing files
+        if (!newProgramFiles.has(forwardSlashPath(dependency))) {
+          this._typeDeps.delete(dependency);
+        }
+      }
+
+      for (const fileName of newProgramFiles) {
+        if (oldFiles.has(fileName)) {
+          continue;
+        }
+
+        this._compilerHost.invalidate(fileName);
+
+        const denormalizedFileName = this._compilerHost.denormalizePath(fileName);
+        if (localDtsFiles.has(denormalizedFileName)) {
+          // Add new dts file as a type depedency
+          this._typeDeps.add(denormalizedFileName);
+        }
       }
     }
 
@@ -630,8 +661,7 @@ export class AngularCompilerPlugin {
 
     // This function removes a source file name and all its dependencies from the set.
     const removeSourceFile = (fileName: string, originalModule = false) => {
-      if (unusedSourceFileNames.has(fileName)
-        || (originalModule && typeDepFileNames.has(fileName))) {
+      if (unusedSourceFileNames.has(fileName) || (originalModule && typeDepFileNames.has(fileName))) {
         unusedSourceFileNames.delete(fileName);
         if (originalModule) {
           typeDepFileNames.delete(fileName);
@@ -649,7 +679,7 @@ export class AngularCompilerPlugin {
       compilation.warnings.push(
         `${fileName} is part of the TypeScript compilation but it's unused.\n` +
         `Add only entry points to the 'files' or 'include' properties in your tsconfig.`,
-        );
+      );
       this._unusedFiles.add(fileName);
       // Remove the truly unused from the type dep list.
       typeDepFileNames.delete(fileName);
@@ -659,7 +689,9 @@ export class AngularCompilerPlugin {
     // These are the TS files that weren't part of the compilation modules, aren't unused, but were
     // part of the TS original source list.
     // Next build we add them to the TS entry points so that they trigger rebuilds.
-    this._typeDeps = typeDepFileNames;
+    for (const fileName of typeDepFileNames) {
+      this._typeDeps.add(fileName);
+    }
   }
 
   // Registration hook for webpack plugin.
@@ -1233,7 +1265,7 @@ export class AngularCompilerPlugin {
 
       for (const resource of resourceImports) {
         for (const dep of this.getResourceDependencies(
-            this._compilerHost.denormalizePath(resource))) {
+          this._compilerHost.denormalizePath(resource))) {
           resourceDependencies.push(dep);
         }
       }