fix(java_common): fixes and utilities for exploded system directories (…

…#4242)

* fix(java_common): throw NotDIrectoryException when listing is a file

Fixes #4213.

* feat(java_extractor): add utility for expanding system modules

* fix(java_indexer): fix indexer support for exploded system image

* chore: rearrange main method

kythe/java/com/google/devtools/kythe/extractors/java/BUILD

@@ -34,3 +34,19 @@ java_library(
         "@maven//:org_checkerframework_checker_qual",
     ],
 )
+
+java_library(
+    name = "system_exploder",
+    srcs = ["SystemExploder.java"],
+    deps = [
+        "//kythe/java/com/google/devtools/kythe/platform/java/filemanager:forwarding_standard_java_file_manager",
+        "//third_party/guava",
+        "//third_party/javac",
+    ],
+)
+
+java_binary(
+    name = "explodesys",
+    main_class = "com.google.devtools.kythe.extractors.java.SystemExploder",
+    runtime_deps = [":system_exploder"],
+)

kythe/java/com/google/devtools/kythe/extractors/java/SystemExploder.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2019 The Kythe Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.devtools.kythe.extractors.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterators;
+import com.google.devtools.kythe.platform.java.filemanager.ForwardingStandardJavaFileManager;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Set;
+import java.util.stream.Stream;
+import javax.tools.JavaFileManager;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.StandardLocation;
+import javax.tools.ToolProvider;
+
+/** Routines for expanding and copying a Java `--system` directory. */
+final class SystemExploder {
+
+  private SystemExploder() {}
+
+  // Copies the system modules from the specified --system directory to the outputPath.
+  public static void copySystemModules(String systemDir, Path outputPath) throws IOException {
+    try (Stream<Path> paths = walkSystemModules(systemDir)) {
+      for (Path path : (Iterable<Path>) paths::iterator) {
+        Files.copy(path, outputPath.resolve(path.subpath(0, path.getNameCount()).toString()));
+      }
+    }
+  }
+
+  public static void copySystemModules(String systemDir, String outputDir) throws IOException {
+    copySystemModules(systemDir, Paths.get(outputDir));
+  }
+
+  // Returns a stream of paths from the specified --system directory, beginning with the appropriate
+  // modules subdirectory.
+  public static Stream<Path> walkSystemModules(String systemDir) throws IOException {
+    StandardJavaFileManager fileManager =
+        ToolProvider.getSystemJavaCompiler().getStandardFileManager(null, null, null);
+    fileManager.handleOption("--system", Iterators.singletonIterator(systemDir));
+    return walkSystemModules(wrap(fileManager));
+  }
+
+  // In order to work around the lack of the required JDK9 methods in google3, wrap
+  // StandardJavaFileManagers before use (for now).
+  private static ForwardingStandardJavaFileManager wrap(StandardJavaFileManager fileManager) {
+    if (fileManager instanceof ForwardingStandardJavaFileManager) {
+      return (ForwardingStandardJavaFileManager) fileManager;
+    }
+    return new ForwardingStandardJavaFileManager(fileManager) {};
+  }
+
+  private static Stream<Path> walkSystemModules(ForwardingStandardJavaFileManager fileManager)
+      throws IOException {
+    JavaFileManager.Location systemLocation = StandardLocation.valueOf("SYSTEM_MODULES");
+    boolean started = false;
+    ImmutableList.Builder<Path> modules = new ImmutableList.Builder<>();
+    for (Set<JavaFileManager.Location> locs : fileManager.listLocationsForModules(systemLocation)) {
+      for (JavaFileManager.Location loc : locs) {
+        for (Path dir : fileManager.getLocationAsPaths(loc)) {
+          try (Stream<Path> stream = Files.walk(dir)) {
+            for (Path path : (Iterable<Path>) stream::iterator) {
+              // Ensure that we include the top-level "modules" directory.
+              if (!started) {
+                started = true;
+                modules.add(path.getParent());
+              }
+              modules.add(path);
+            }
+          }
+        }
+      }
+    }
+    // TODO(shahms): make this a lazy stream.
+    return modules.build().stream();
+  }
+
+  public static void main(String[] args) {
+    try {
+      if (args.length == 2) {
+        copySystemModules(args[0], args[1]);
+      } else if (args.length == 1) {
+        dumpSystemPaths(args[0]);
+      } else {
+        dumpSystemPaths();
+      }
+    } catch (IOException e) {
+      System.err.println(e.toString());
+    }
+  }
+
+  private static void dumpSystemPaths(ForwardingStandardJavaFileManager fileManager)
+      throws IOException {
+    try (Stream<Path> paths = walkSystemModules(fileManager)) {
+      for (Path path : (Iterable<Path>) paths::iterator) {
+        System.err.println(
+            (path.isAbsolute() ? path.subpath(0, path.getNameCount()) : path).toString());
+      }
+    }
+  }
+
+  private static void dumpSystemPaths() throws IOException {
+    dumpSystemPaths(
+        wrap(ToolProvider.getSystemJavaCompiler().getStandardFileManager(null, null, null)));
+  }
+
+  private static void dumpSystemPaths(String systemDir) throws IOException {
+    StandardJavaFileManager fileManager =
+        ToolProvider.getSystemJavaCompiler().getStandardFileManager(null, null, null);
+    fileManager.handleOption("--system", Iterators.singletonIterator(systemDir));
+    dumpSystemPaths(wrap(fileManager));
+  }
+}

kythe/java/com/google/devtools/kythe/platform/java/filemanager/CompilationUnitPathFileManager.java

@@ -33,6 +33,7 @@
 import com.sun.tools.javac.main.Option;
 import com.sun.tools.javac.main.OptionHelper;
 import java.io.IOException;
+import java.nio.file.DirectoryStream;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -259,10 +260,15 @@ private void setSystemOption(String value) throws IOException {
       logger.atInfo().log("Setting system path to %s", systemRoot);
       super.handleOption("--system", Iterators.singletonIterator(systemRoot.toString()));
     } else if (Files.isDirectory(sys.resolve("modules"))) {
-      // TODO(salguarnieri) Due to a bug in the javac argument validation, bypass it and set the
-      // location directly.
-      setLocationFromPaths(
-          StandardLocation.valueOf("SYSTEM_MODULES"), ImmutableList.of(sys.resolve("modules")));
+      // TODO(shahms): Due to a bug in both javac argument validation and location validation,
+      // we have to manually enumerate the available modules and set them directly.
+      Location systemLocation = StandardLocation.valueOf("SYSTEM_MODULES");
+      try (DirectoryStream<Path> stream = Files.newDirectoryStream(sys.resolve("modules"))) {
+        for (Path entry : stream) {
+          setLocationForModule(
+              systemLocation, entry.getFileName().toString(), ImmutableList.of(entry));
+        }
+      }
     } else {
       throw new IllegalArgumentException(value);
     }

kythe/java/com/google/devtools/kythe/platform/shared/filesystem/CompilationUnitFileSystem.java

@@ -28,6 +28,7 @@
 import java.nio.file.FileStore;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
+import java.nio.file.NotDirectoryException;
 import java.nio.file.Path;
 import java.nio.file.PathMatcher;
 import java.nio.file.WatchService;
@@ -193,6 +194,9 @@ Iterable<Path> list(Path dir) throws IOException {
     final Path abs = getRootDirectory().resolve(dir).normalize();
     Map<String, String> entries = compilationFileTree.list(abs.toString());
     if (entries == null) {
+      if (!CompilationUnitFileTree.DIRECTORY_DIGEST.equals(digest(abs))) {
+        throw new NotDirectoryException(dir.toString());
+      }
       throw new FileNotFoundException(dir.toString());
     }
     return entries.keySet().stream().map(k -> dir.resolve(k)).collect(Collectors.toSet());

kythe/javatests/com/google/devtools/kythe/platform/shared/filesystem/CompilationUnitFileSystemTest.java

@@ -17,6 +17,7 @@
 package com.google.devtools.kythe.platform.shared.filesystem;
 
 import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.fail;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.kythe.extractors.shared.ExtractorUtils;
@@ -26,7 +27,9 @@
 import com.google.devtools.kythe.proto.Analysis.FileData;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
+import java.nio.file.NotDirectoryException;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -222,6 +225,25 @@ public void filesWalk_readsAllFiles() {
     }
   }
 
+  @Test
+  public void newDirectoryStream_failsOnNonDirectory() {
+    CompilationUnitFileSystem fileSystem =
+        builder()
+            .addFile("relative/nested/path/with/empty/file", "relativeContents")
+            .addFile("/absolute/nested/path/with/empty/file", "absoluteContents")
+            .build();
+
+    try {
+      try (DirectoryStream<Path> stream =
+          Files.newDirectoryStream(fileSystem.getPath("/absolute/nested/path/with/empty/file"))) {}
+      fail("Expected NotDirectoryException not thrown.");
+    } catch (NotDirectoryException exc) {
+      assertThat(exc).hasMessageThat().isEqualTo("/absolute/nested/path/with/empty/file");
+    } catch (IOException exc) {
+      throw new RuntimeException(exc);
+    }
+  }
+
   @Test
   public void toRealPath_usesCompilationRoot() {
     CompilationUnitFileSystem fileSystem =