replace list files in Indexer with RESTful API endpoint

Vladimir Kotal · Vladimir Kotal · commit 7516a8e82680 · 2019-02-05T20:27:51.000+01:00
diff --git a/apiary.apib b/apiary.apib
@@ -172,6 +172,15 @@ This entry point is used by the Indexer once it finishes indexing given project.
 
 + Response 204
 
+## Project index files [/projects/{project}/files]
+
+### Get a list of files tracked by the index database for given project [GET]
+
++ Response 200 (application/json)
+  + Body
+
+            ["/project/foo.txt","/project/bar.txt"]
+
 ## Project metadata [/projects/{project}/property/{field}]
 
 ### sets property field for the project [PUT]
diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java
@@ -1312,10 +1312,9 @@ public static Set<String> getAllFiles() throws IOException {
     }
 
     /**
-     * List all files in some of the index databases.
+     * Get all files in some of the index databases.
      *
-     * @param subFiles Subdirectories for the various projects to list the files
-     * for (or null or an empty list to dump all projects)
+     * @param subFiles Subdirectories of various projects or null or an empty list to get everything
      * @throws IOException if an error occurs
      * @return set of files in the index databases specified by the subFiles parameter
      */
@@ -1369,7 +1368,13 @@ public Set<String> getFiles() throws IOException {
                 iter = terms.iterator(); // init uid iterator
             }
             while (iter != null && iter.term() != null) {
-                files.add(Util.uid2url(iter.term().utf8ToString()));
+                String value = iter.term().utf8ToString();
+                if (value.isEmpty()) {
+                    iter.next();
+                    continue;
+                }
+
+                files.add(Util.uid2url(value));
                 BytesRef next = iter.next();
                 if (next == null) {
                     iter = null;
diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java
@@ -136,7 +136,6 @@ public static void main(String[] argv) {
         ArrayList<String> subFiles = new ArrayList<>();
         ArrayList<String> subFilesList = new ArrayList<>();
 
-        boolean listFiles = false;
         boolean createDict = false;
 
         try {
@@ -315,7 +314,7 @@ public static void main(String[] argv) {
             // Create history cache first.
             getInstance().prepareIndexer(env, searchRepositories, addProjects,
                     defaultProjects,
-                    listFiles, createDict, runIndex, subFiles, new ArrayList<>(repositories));
+                    createDict, runIndex, subFiles, new ArrayList<>(repositories));
 
             // And now index it all.
             if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {
@@ -877,11 +876,11 @@ public void prepareIndexer(RuntimeEnvironment env,
                                boolean searchRepositories,
                                boolean addProjects,
                                Set<String> defaultProjects,
-                               boolean listFiles,
                                boolean createDict,
                                List<String> subFiles,
                                List<String> repositories) throws IndexerException, IOException {
-        prepareIndexer(env, searchRepositories, addProjects, defaultProjects, listFiles, createDict, true,
+
+        prepareIndexer(env, searchRepositories, addProjects, defaultProjects, createDict, true,
                 subFiles, repositories);
     }
 
@@ -899,7 +898,6 @@ public void prepareIndexer(RuntimeEnvironment env,
      * @param searchRepositories if true, search for repositories
      * @param addProjects if true, add projects
      * @param defaultProjects default projects
-     * @param listFiles list files and return
      * @param createDict if true, create dictionary
      * @param createHistoryCache create history cache flag
      * @param subFiles list of directories
@@ -912,7 +910,6 @@ public void prepareIndexer(RuntimeEnvironment env,
             boolean searchRepositories,
             boolean addProjects,
             Set<String> defaultProjects,
-            boolean listFiles,
             boolean createDict,
             boolean createHistoryCache,
             List<String> subFiles,
@@ -974,14 +971,6 @@ public void prepareIndexer(RuntimeEnvironment env,
                     env.getRepositories().size()));
         }
 
-        if (listFiles) {
-            for (String file : IndexDatabase.getAllFiles(subFiles)) {
-                LOGGER.fine(file);
-            }
-
-            return;
-        }
-
         if (defaultProjects != null && !defaultProjects.isEmpty()) {
             Set<Project> projects = new TreeSet<>();
             for (String projectPath : defaultProjects) {
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java
@@ -85,7 +85,7 @@ public static void setUpClass() throws Exception {
         IndexChangedListener progress = new DefaultIndexChangedListener();
         Indexer.getInstance().prepareIndexer(env, true, true,
                 new TreeSet<>(Collections.singletonList("/c")),
-                false, false, null, null);
+                false, null, null);
 
         Indexer.getInstance().doIndexerExecution(true, null, progress);
 
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java
@@ -88,7 +88,7 @@ public static void setUpClass() throws Exception {
         Indexer indexer = Indexer.getInstance();
         indexer.prepareIndexer(
                 env, true, true, new TreeSet<>(Arrays.asList(new String[]{"/c"})),
-                false, false, null, null);
+                false, null, null);
         indexer.doIndexerExecution(true, null, null);
     }
 
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexVersionTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexVersionTest.java
@@ -91,7 +91,7 @@ private void testIndexVersion(boolean projectsEnabled, List<String> subFiles) th
         env.setHistoryEnabled(false);
         env.setProjectsEnabled(projectsEnabled);
         Indexer.getInstance().prepareIndexer(env, true, true, null,
-                false, false, null, null);
+                false, null, null);
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
         IndexVersion.check(subFiles);
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java
@@ -139,7 +139,6 @@ private void testPerProjectHistory(boolean globalOn) throws IndexerException, IO
                 true, // search for repositories
                 true, // scan and add projects
                 null, // no default project
-                false, // don't list files
                 false, // don't create dictionary
                 null, // subFiles - not needed since we don't list files
                 null); // repositories - not needed when not refreshing history
@@ -203,7 +202,6 @@ public void testSymlinks() throws IndexerException, IOException {
                 true, // search for repositories
                 true, // scan and add projects
                 null, // no default project
-                false, // don't list files
                 false, // don't create dictionary
                 null, // subFiles - not needed since we don't list files
                 null); // repositories - not needed when not refreshing history
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java
@@ -121,7 +121,7 @@ public void testIndexGeneration() throws Exception {
         env.setDataRoot(repository.getDataRoot());
         env.setHistoryEnabled(false);
         Indexer.getInstance().prepareIndexer(env, true, true, new TreeSet<>(Collections.singletonList("/c")),
-                false, false, null, null);
+                false, null, null);
         Indexer.getInstance().doIndexerExecution(true, null, null);
     }
 
@@ -155,7 +155,6 @@ public void testRescanProjects() throws Exception {
                 false, // don't search for repositories
                 true, // scan and add projects
                 null, // no default project
-                false, // don't list files
                 false, // don't create dictionary
                 null, // subFiles - not needed since we don't list files
                 null); // repositories - not needed when not refreshing history
@@ -485,7 +484,7 @@ public void testDefaultProjectsSingleProject() throws Exception {
         Indexer.getInstance().prepareIndexer(env, true, true, new TreeSet<>(Arrays.asList(new String[]{
                         Paths.get("/c").toString()
                 })),
-                false, false, null, null);
+                false, null, null);
         assertEquals(1, env.getDefaultProjects().size());
         assertEquals(new TreeSet<>(Arrays.asList(new String[]{"c"})),
                 env.getDefaultProjects().stream().map((Project p) -> p.getName()).collect(Collectors.toSet()));
@@ -511,7 +510,7 @@ public void testDefaultProjectsNonExistent() throws Exception {
                         Paths.get("/data").toString(),
                         Paths.get("/no-project-x32ds1").toString()
                 })),
-                false, false, null, null);
+                false, null, null);
         assertEquals(4, env.getDefaultProjects().size());
         assertEquals(new TreeSet<>(Arrays.asList(new String[]{"lisp", "pascal", "perl", "data"})),
                 env.getDefaultProjects().stream().map((Project p) -> p.getName()).collect(Collectors.toSet()));
@@ -535,7 +534,7 @@ public void testDefaultProjectsAll() throws Exception {
                         Paths.get("__all__").toString(),
                         Paths.get("/no-project-x32ds1").toString()
                 })),
-                false, false, null, null);
+                false, null, null);
         Set<String> projects = new TreeSet<>(Arrays.asList(new File(repository.getSourceRoot()).list()));
         assertEquals(projects.size(), env.getDefaultProjects().size());
         assertEquals(projects, env.getDefaultProjects().stream().map((Project p) -> p.getName()).collect(Collectors.toSet()));
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java
@@ -75,7 +75,7 @@ public static void setUpClass() throws Exception {
         env.setHistoryEnabled(false);
         Indexer.getInstance().prepareIndexer(env, true, true,
                 new TreeSet<>(Collections.singletonList("/c")),
-                false, false, null, null);
+                false, null, null);
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
 
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java
@@ -89,7 +89,7 @@ public static void setUpClass() throws Exception {
         env.setHistoryEnabled(false);
         Indexer.getInstance().prepareIndexer(env, true, true,
                 new TreeSet<>(Collections.singletonList("/c")),
-                false, false, null, null);
+                false, null, null);
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
         configFile = File.createTempFile("configuration", ".xml");
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java
@@ -128,7 +128,7 @@ public static void setUpClass() throws Exception {
         env.setHistoryEnabled(false);
         Indexer.getInstance().prepareIndexer(env, true, true,
                 new TreeSet<>(Collections.singletonList("/c")),
-                false, false, null, null);
+                false, null, null);
 
         Project proj1 = env.getProjects().get(SYMLINK1);
         assertNotNull("symlink1 project", proj1);
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java
@@ -76,7 +76,7 @@ private void reindex() throws Exception {
 
         Indexer.getInstance().prepareIndexer(env, true, true,
             new TreeSet<>(Collections.singletonList("/c")),
-            false, false, null, null);
+            false, null, null);
         Indexer.getInstance().doIndexerExecution(true, null, null);
     }
 
diff --git a/opengrok-web/src/main/java/org/opengrok/web/api/v1/controller/ProjectsController.java b/opengrok-web/src/main/java/org/opengrok/web/api/v1/controller/ProjectsController.java
@@ -352,4 +352,10 @@ public Set<String> getRepositoriesType(@PathParam("project") final String projec
         return Collections.emptySet();
     }
 
+    @GET
+    @Path("/{project}/files")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Set<String> getProjectIndexFiles(@PathParam("project") String projectName) throws IOException {
+        return IndexDatabase.getAllFiles(Collections.singletonList("/" + projectName));
+    }
 }
diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java
@@ -45,6 +45,7 @@
 import org.opengrok.indexer.history.RepositoryInfo;
 import org.opengrok.indexer.index.IndexDatabase;
 import org.opengrok.indexer.index.Indexer;
+import org.opengrok.indexer.index.IndexerException;
 import org.opengrok.indexer.util.TestRepository;
 import org.opengrok.web.api.v1.suggester.provider.service.SuggesterService;
 
@@ -310,7 +311,6 @@ public void testDelete() throws Exception {
                 false, // don't search for repositories
                 false, // don't scan and add projects
                 null, // no default project
-                false, // don't list files
                 false, // don't create dictionary
                 subFiles, // subFiles - needed when refreshing history partially
                 repos); // repositories - needed when refreshing history partially
@@ -539,4 +539,34 @@ private void setHandleRenamedFilesToFalse(final String project) {
                 .put(Entity.text(Boolean.FALSE.toString()));
     }
 
+    @Test
+    public void testListFiles() throws IOException, IndexerException {
+        final String projectName = "mercurial";
+        GenericType<List<String>> type = new GenericType<List<String>>() {};
+
+        Indexer.getInstance().prepareIndexer(
+                env,
+                false, // don't search for repositories
+                true, // add projects
+                null, // no default project
+                false, // don't create dictionary
+                new ArrayList<>(), // subFiles - needed when refreshing history partially
+                new ArrayList<>()); // repositories - needed when refreshing history partially
+        Indexer.getInstance().doIndexerExecution(true, null, null);
+
+        List<String> filesFromRequest = target("projects")
+                .path(projectName)
+                .path("files")
+                .request()
+                .get(type);
+        filesFromRequest.sort(String::compareTo);
+        String files[] = {"Makefile", "bar.txt", "header.h", "main.c", "novel.txt"};
+        for (int i = 0; i < files.length; i++) {
+            files[i] = "/" + projectName + "/" + files[i];
+        }
+        List<String> expectedFiles = Arrays.asList(files);
+        expectedFiles.sort(String::compareTo);
+
+        assertEquals(expectedFiles, filesFromRequest);
+    }
 }
diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/RepositoriesControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/RepositoriesControllerTest.java
@@ -108,7 +108,6 @@ public void testGetRepositoryType() throws Exception {
                 true, // search for repositories
                 true, // scan and add projects
                 null, // no default project
-                false, // don't list files
                 false, // don't create dictionary
                 null, // subFiles - needed when refreshing history partially
                 null); // repositories - needed when refreshing history partially
diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java
@@ -80,7 +80,7 @@ public static void setUpClass() throws Exception {
 
         Indexer.getInstance().prepareIndexer(env, true, true,
                 Collections.singleton("__all__"),
-                false, false, null, null);
+                false, null, null);
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
         env.getSuggesterConfig().setRebuildCronConfig(null);
diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java
@@ -112,7 +112,7 @@ public static void setUpClass() throws Exception {
         env.setProjectsEnabled(true);
         Indexer.getInstance().prepareIndexer(env, true, true,
                 Collections.singleton("__all__"),
-                false, false, null, null);
+                false, null, null);
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
         env.getSuggesterConfig().setRebuildCronConfig(null);

Original file line number	Diff line number	Diff line change
`@@ -88,7 +88,7 @@ public static void setUpClass() throws Exception {`
`88`	`88`	`Indexer indexer = Indexer.getInstance();`
`89`	`89`	`indexer.prepareIndexer(`
`90`	`90`	`env, true, true, new TreeSet<>(Arrays.asList(new String[]{"/c"})),`
`91`		`- false, false, null, null);`
	`91`	`+ false, null, null);`
`92`	`92`	`indexer.doIndexerExecution(true, null, null);`
`93`	`93`	`}`
`94`	`94`