YARN-9255. Improve recommend applications order and fix findbugs warnings. Contributed by Eric Yang

Author: billierinaldi

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/AppCatalogSolrClient.java

@@ -38,6 +38,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.response.QueryResponse;
@@ -64,12 +65,16 @@ public AppCatalogSolrClient() {
     Properties properties = new Properties();
     try {
       properties.load(input);
-      urlString = properties.getProperty("solr_url");
+      setSolrUrl(properties.getProperty("solr_url"));
     } catch (IOException e) {
       LOG.error("Error reading appcatalog configuration: ", e);
     }
   }
 
+  private synchronized static void setSolrUrl(String url) {
+    urlString = url;
+  }
+
   public SolrClient getSolrClient() {
     return new HttpSolrClient.Builder(urlString).build();
   }
@@ -79,6 +84,7 @@ public List<AppStoreEntry> getRecommendedApps() {
     SolrClient solr = getSolrClient();
     SolrQuery query = new SolrQuery();
     query.setQuery("*:*");
+    query.setSort("download_i", ORDER.desc);
     query.setFilterQueries("type_s:AppStoreEntry");
     query.setRows(40);
     QueryResponse response;
@@ -95,8 +101,8 @@ public List<AppStoreEntry> getRecommendedApps() {
         if (d.get("icon_s")!=null) {
           entry.setIcon(d.get("icon_s").toString());
         }
-        entry.setLike(Integer.valueOf(d.get("like_i").toString()));
-        entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
+        entry.setLike(Integer.parseInt(d.get("like_i").toString()));
+        entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
         apps.add(entry);
       }
     } catch (SolrServerException | IOException e) {
@@ -128,8 +134,8 @@ public List<AppStoreEntry> search(String keyword) {
         entry.setOrg(d.get("org_s").toString());
         entry.setName(d.get("name_s").toString());
         entry.setDesc(d.get("desc_s").toString());
-        entry.setLike(Integer.valueOf(d.get("like_i").toString()));
-        entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
+        entry.setLike(Integer.parseInt(d.get("like_i").toString()));
+        entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
         apps.add(entry);
       }
     } catch (SolrServerException | IOException e) {
@@ -189,8 +195,8 @@ public AppStoreEntry findAppStoreEntry(String id) {
         entry.setOrg(d.get("org_s").toString());
         entry.setName(d.get("name_s").toString());
         entry.setDesc(d.get("desc_s").toString());
-        entry.setLike(Integer.valueOf(d.get("like_i").toString()));
-        entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
+        entry.setLike(Integer.parseInt(d.get("like_i").toString()));
+        entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
         Service yarnApp = mapper.readValue(d.get("yarnfile_s").toString(),
             Service.class);
         String name;
@@ -263,8 +269,8 @@ public void deployApp(String id, Service service) throws SolrServerException,
       entry.setOrg(d.get("org_s").toString());
       entry.setName(d.get("name_s").toString());
       entry.setDesc(d.get("desc_s").toString());
-      entry.setLike(Integer.valueOf(d.get("like_i").toString()));
-      entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
+      entry.setLike(Integer.parseInt(d.get("like_i").toString()));
+      entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
       download = entry.getDownload() + 1;
 
       // Update download count
@@ -303,7 +309,8 @@ private SolrInputDocument incrementDownload(SolrDocument doc,
         s.addField(name, doc.getFieldValues(name));
       }
     }
-    s.setField("download_i", download++);
+    download++;
+    s.setField("download_i", download);
     return s;
   }
 
@@ -356,4 +363,42 @@ public void register(Application app) throws IOException {
     }
   }
 
+  protected void register(AppStoreEntry app) throws IOException {
+    Collection<SolrInputDocument> docs = new HashSet<SolrInputDocument>();
+    SolrClient solr = getSolrClient();
+    ObjectMapper mapper = new ObjectMapper();
+    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    try {
+      SolrInputDocument buffer = new SolrInputDocument();
+      buffer.setField("id", java.util.UUID.randomUUID().toString()
+          .substring(0, 11));
+      buffer.setField("org_s", app.getOrg());
+      buffer.setField("name_s", app.getName());
+      buffer.setField("desc_s", app.getDesc());
+      if (app.getIcon() != null) {
+        buffer.setField("icon_s", app.getIcon());
+      }
+      buffer.setField("type_s", "AppStoreEntry");
+      buffer.setField("like_i", app.getLike());
+      buffer.setField("download_i", app.getDownload());
+
+      // Keep only YARN data model for yarnfile field
+      String yarnFile = mapper.writeValueAsString(app);
+      LOG.info("app:"+yarnFile);
+      Service yarnApp = mapper.readValue(yarnFile, Service.class);
+      buffer.setField("yarnfile_s", mapper.writeValueAsString(yarnApp));
+
+      docs.add(buffer);
+      // Commit Solr changes.
+      UpdateResponse detailsResponse = solr.add(docs);
+      if (detailsResponse.getStatus() != 0) {
+        throw new IOException("Unable to register application " +
+            "in Application Store.");
+      }
+      solr.commit();
+    } catch (SolrServerException | IOException e) {
+      throw new IOException("Unable to register application " +
+          "in Application Store. "+ e.getMessage());
+    }
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java

@@ -51,26 +51,26 @@ public void setVersion(String version) {
   }
 
   public String[] getPorts() {
-    return ports;
+    return ports.clone();
   }
 
   public void setPorts(String[] ports2) {
-    this.ports = ports2;
+    this.ports = ports2.clone();
   }
 
   public String[] getVolumes() {
-    return volumes;
+    return volumes.clone();
   }
 
   public void setVolumes(String[] volumes) {
-    this.volumes = volumes;
+    this.volumes = volumes.clone();
   }
 
   public String[] getEnv() {
-    return env;
+    return env.clone();
   }
 
   public void setEnv(String[] env) {
-    this.env = env;
+    this.env = env.clone();
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.appcatalog.model;
 
+import java.util.Objects;
+
 import javax.xml.bind.annotation.XmlRootElement;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@@ -64,4 +66,24 @@ public void setIcon(String icon) {
     this.icon = icon;
   }
 
+  @Override
+  public boolean equals(final Object obj) {
+    if (obj == this) {
+      return true;
+    } else if ((obj instanceof Application)) {
+      if (((Application) obj).getName().equals(this.getName())
+          && ((Application) obj).getVersion().equals(this.getVersion())
+          && ((Application) obj).getOrganization()
+              .equals(this.getOrganization())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects
+        .hash(this.getName() + this.getVersion() + this.getOrganization());
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java

@@ -104,9 +104,27 @@ public void testNotFoundSearch() throws Exception {
 
   @Test
   public void testGetRecommendedApps() throws Exception {
-    List<AppStoreEntry> expected = spy.getRecommendedApps();
+    AppStoreEntry example = new AppStoreEntry();
+    example.setOrg("jenkins-ci.org");
+    example.setName("jenkins");
+    example.setDesc("World leading open source automation system.");
+    example.setIcon("/css/img/feather.png");
+    example.setDownload(100);
+    spy.register(example);
+    AppStoreEntry example2 = new AppStoreEntry();
+    example2.setOrg("Apache");
+    example2.setName("httpd");
+    example2.setDesc("Apache webserver");
+    example2.setIcon("/css/img/feather.png");
+    example2.setDownload(1);
+    spy.register(example2);
     List<AppStoreEntry> actual = spy.getRecommendedApps();
-    assertEquals(expected, actual);
+    long previous = 1000L;
+    for (AppStoreEntry app: actual) {
+      assertTrue("Recommend app is not sort by download count.",
+          previous > app.getDownload());
+      previous = app.getDownload();
+    }
   }
 
 }