Merge pull request #2582 from idodeclare/feature/history_ctags

Support ctags for historical revisions

Author: tarzanek

dev/checkstyle/suppressions.xml

@@ -1,5 +1,27 @@
 <?xml version="1.0"?>
+<!--
 
+CDDL HEADER START
+
+The contents of this file are subject to the terms of the
+Common Development and Distribution License (the "License").
+You may not use this file except in compliance with the License.
+
+See LICENSE.txt included in this distribution for the specific
+language governing permissions and limitations under the License.
+
+When distributing Covered Code, include this CDDL HEADER in each
+file and include the License file at LICENSE.txt.
+If applicable, add the following below this CDDL HEADER, with the
+fields enclosed by brackets "[]" replaced with your own identifying
+information: Portions Copyright [yyyy] [name of copyright owner]
+
+CDDL HEADER END
+
+Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+Portions Copyright (c) 2018, Chris Fraire <[email protected]>.
+
+-->
 <!DOCTYPE suppressions PUBLIC
         "-//Checkstyle//DTD SuppressionFilter Configuration 1.1//EN"
         "https://checkstyle.org/dtds/suppressions_1_1.dtd">
@@ -9,7 +31,8 @@
         |CustomExactPhraseScorer\.java|PhrasePositions\.java|PhraseQueue\.java|Summarizer\.java|
         |Summary\.java|OGKUnifiedHighlighter\.java|ResponseHeaderFilter\.java|BoundedBlockingObjectPool\.java|
         |ObjectPool\.java|ObjectValidator\.java|ObjectFactory\.java|AbstractObjectPool\.java|
-        |BlockingObjectPool\.java|OGKTextVecField\.java|OGKTextField\.java" />
+        |BlockingObjectPool\.java|OGKTextVecField\.java|OGKTextField\.java|
+        |LazilyInstantiate\.java" />
 
     <suppress checks="ParameterNumber" files="CtagsReader\.java|Definitions\.java|PerlLexHelper\.java|
         |JFlexXrefUtils\.java|RubyLexHelper\.java|FileAnalyzerFactory\.java|SearchController\.java|

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -107,6 +107,7 @@ public final class Configuration {
      * Path to {@code ctags} binary.
      */
     private String ctags;
+    private boolean webappCtags;
 
     /**
      * A defined value to specify the mandoc binary or else null so that mandoc
@@ -475,6 +476,7 @@ public Configuration() {
         // unconditionally later.
         setUserPageSuffix("");
         setWebappLAF("default");
+        // webappCtags is default(boolean)
     }
 
     public String getRepoCmd(String clazzName) {
@@ -983,6 +985,20 @@ public void setWebappLAF(String webappLAF) {
         this.webappLAF = webappLAF;
     }
 
+    /**
+     * Gets a value indicating if the web app should run ctags as necessary.
+     */
+    public boolean isWebappCtags() {
+        return webappCtags;
+    }
+
+    /**
+     * Sets a value indicating if the web app should run ctags as necessary.
+     */
+    public void setWebappCtags(boolean value) {
+        this.webappCtags = value;
+    }
+
     public RemoteSCM getRemoteScmSupported() {
         return remoteScmSupported;
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -43,7 +43,6 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.logging.Level;
@@ -65,9 +64,11 @@
 import org.opengrok.indexer.index.Filter;
 import org.opengrok.indexer.index.IgnoredNames;
 import org.opengrok.indexer.index.IndexDatabase;
+import org.opengrok.indexer.index.IndexerParallelizer;
 import org.opengrok.indexer.logger.LoggerFactory;
 import org.opengrok.indexer.util.CtagsUtil;
 import org.opengrok.indexer.util.ForbiddenSymlinkException;
+import org.opengrok.indexer.util.LazilyInstantiate;
 import org.opengrok.indexer.util.PathUtils;
 import org.opengrok.indexer.web.Prefix;
 import org.opengrok.indexer.web.Statistics;
@@ -88,11 +89,10 @@ public final class RuntimeEnvironment {
     private static final String URL_PREFIX = "/source" + Prefix.SEARCH_R + "?";
 
     private Configuration configuration;
-    private ReentrantReadWriteLock configLock;
+    private final ReentrantReadWriteLock configLock;
+    private final LazilyInstantiate<IndexerParallelizer> lzIndexerParallelizer;
+    private final LazilyInstantiate<ExecutorService> lzSearchExecutor;
     private static final RuntimeEnvironment instance = new RuntimeEnvironment();
-    private static ExecutorService historyExecutor = null;
-    private static ExecutorService historyRenamedExecutor = null;
-    private static ExecutorService searchExecutor = null;
 
     private final Map<Project, List<RepositoryInfo>> repository_map = new ConcurrentHashMap<>();
     private final Map<String, SearcherManager> searcherManagerMap = new ConcurrentHashMap<>();
@@ -127,43 +127,21 @@ private RuntimeEnvironment() {
         configuration = new Configuration();
         configLock = new ReentrantReadWriteLock();
         watchDog = new WatchDogService();
+        lzIndexerParallelizer = LazilyInstantiate.using(() ->
+                new IndexerParallelizer(this));
+        lzSearchExecutor = LazilyInstantiate.using(() -> newSearchExecutor());
     }
 
     /** Instance of authorization framework.*/
     private AuthorizationFramework authFramework;
 
-    /* Get thread pool used for top-level repository history generation. */
-    public static synchronized ExecutorService getHistoryExecutor() {
-        if (historyExecutor == null) {
-            historyExecutor = Executors.newFixedThreadPool(getInstance().getHistoryParallelism(),
-                    runnable -> {
-                        Thread thread = Executors.defaultThreadFactory().newThread(runnable);
-                        thread.setName("history-handling-" + thread.getId());
-                        return thread;
-                    });
-        }
-
-        return historyExecutor;
-    }
-
-    /* Get thread pool used for history generation of renamed files. */
-    public static synchronized ExecutorService getHistoryRenamedExecutor() {
-        if (historyRenamedExecutor == null) {
-            historyRenamedExecutor = Executors.newFixedThreadPool(getInstance().getHistoryRenamedParallelism(),
-                    runnable -> {
-                        Thread thread = Executors.defaultThreadFactory().newThread(runnable);
-                        thread.setName("renamed-handling-" + thread.getId());
-                        return thread;
-                    });
-        }
-
-        return historyRenamedExecutor;
+    /** Gets the thread pool used for multi-project searches. */
+    public ExecutorService getSearchExecutor() {
+        return lzSearchExecutor.get();
     }
 
-    /* Get thread pool used for multi-project searches. */
-    public synchronized ExecutorService getSearchExecutor() {
-        if (searchExecutor == null) {
-            searchExecutor = Executors.newFixedThreadPool(
+    private ExecutorService newSearchExecutor() {
+        return Executors.newFixedThreadPool(
                 this.getMaxSearchThreadCount(),
                 new ThreadFactory() {
                 @Override
@@ -173,23 +151,6 @@ public Thread newThread(Runnable runnable) {
                     return thread;
                 }
             });
-        }
-
-        return searchExecutor;
-    }
-
-    public static synchronized void freeHistoryExecutor() {
-        historyExecutor = null;
-    }
-
-    public static synchronized void destroyRenamedHistoryExecutor() throws InterruptedException {
-        if (historyRenamedExecutor != null) {
-            historyRenamedExecutor.shutdown();
-            // All the jobs should be completed by now however for testing
-            // we would like to make sure the threads are gone.
-            historyRenamedExecutor.awaitTermination(1, TimeUnit.MINUTES);
-            historyRenamedExecutor = null;
-        }
     }
 
     /**
@@ -203,6 +164,10 @@ public static RuntimeEnvironment getInstance() {
 
     public WatchDogService watchDog;
 
+    public IndexerParallelizer getIndexerParallelizer() {
+        return lzIndexerParallelizer.get();
+    }
+
     private String getCanonicalPath(String s) {
         if (s == null) { return null; }
         try {
@@ -1092,6 +1057,14 @@ public void setWebappLAF(String laf) {
         setConfigurationValue("webappLAF", laf);
     }
 
+    /**
+     * Gets a value indicating if the web app should run ctags as necessary.
+     * @return the value of {@link Configuration#isWebappCtags()}
+     */
+    public boolean isWebappCtags() {
+        return (boolean)getConfigurationValue("webappCtags");
+    }
+
     public Configuration.RemoteSCM getRemoteScmSupported() {
         return (Configuration.RemoteSCM)getConfigurationValue("remoteScmSupported");
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/history/AccuRevRepository.java

@@ -19,14 +19,13 @@
 
 /*
  * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2018, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.history;
 
 import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.nio.file.Path;
@@ -38,6 +37,7 @@
 import java.util.regex.Pattern;
 import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.logger.LoggerFactory;
+import org.opengrok.indexer.util.BufferSink;
 import org.opengrok.indexer.util.Executor;
 
 /**
@@ -165,10 +165,10 @@ Executor getHistoryLogExecutor(File file) throws IOException {
     }
 
     @Override
-    InputStream getHistoryGet(String parent, String basename, String rev) {
+    boolean getHistoryGet(
+            BufferSink sink, String parent, String basename, String rev) {
 
         ArrayList<String> cmd = new ArrayList<>();
-        InputStream inputStream = null;
         File directory = new File(parent);
 
         /*
@@ -216,12 +216,16 @@ InputStream getHistoryGet(String parent, String basename, String rev) {
 
             executor = new Executor(cmd, directory);
             executor.exec();
-
-            inputStream
-                    = new ByteArrayInputStream(executor.getOutputString().getBytes());
+            try {
+                copyBytes(sink, executor.getOutputStream());
+                return true;
+            } catch (IOException e) {
+                LOGGER.log(Level.SEVERE, "Failed to obtain content for {0}",
+                        basename);
+            }
         }
 
-        return inputStream;
+        return false;
     }
 
     @Override

opengrok-indexer/src/main/java/org/opengrok/indexer/history/BazaarRepository.java

@@ -19,22 +19,20 @@
 
 /*
  * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.history;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.logger.LoggerFactory;
+import org.opengrok.indexer.util.BufferSink;
 import org.opengrok.indexer.util.Executor;
 
 /**
@@ -96,31 +94,19 @@ Executor getHistoryLogExecutor(final File file, final String sinceRevision)
     }
 
     @Override
-    public InputStream getHistoryGet(String parent, String basename, String rev) {
-        InputStream ret = null;
+    boolean getHistoryGet(
+            BufferSink sink, String parent, String basename, String rev) {
 
         File directory = new File(getDirectoryName());
-
         Process process = null;
         try {
             String filename = (new File(parent, basename)).getCanonicalPath()
                     .substring(getDirectoryName().length() + 1);
             ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
             String[] argv = {RepoCommand, "cat", "-r", rev, filename};
             process = Runtime.getRuntime().exec(argv, null, directory);
-
-            ByteArrayOutputStream out = new ByteArrayOutputStream();
-            byte[] buffer = new byte[32 * 1024];
-            InputStream in = process.getInputStream();
-            int len;
-
-            while ((len = in.read(buffer)) != -1) {
-                if (len > 0) {
-                    out.write(buffer, 0, len);
-                }
-            }
-
-            ret = new ByteArrayInputStream(out.toByteArray());
+            copyBytes(sink, process.getInputStream());
+            return true;
         } catch (Exception exp) {
             LOGGER.log(Level.SEVERE,
                     "Failed to get history: " + exp.getClass().toString(), exp);
@@ -136,7 +122,7 @@ public InputStream getHistoryGet(String parent, String basename, String rev) {
             }
         }
 
-        return ret;
+        return false;
     }
 
     /**