Index rebuild is done synchronously. Added some logging messages for monitoring

diff --git a/src/gr/ebs/gss/mbeans/Solr.java b/src/gr/ebs/gss/mbeans/Solr.java
index a6a2003..00f0ab3 100644 (file)
--- a/src/gr/ebs/gss/mbeans/Solr.java
+++ b/src/gr/ebs/gss/mbeans/Solr.java
@@ -37,12 +37,12 @@ import org.jboss.system.ServiceMBeanSupport;
 public class Solr extends ServiceMBeanSupport implements SolrMBean {
 
        @Override
-       public void rebuildIndex() {
+       public String rebuildIndex() {
                try {
                        InitialContext ctx = new InitialContext();
                        Object ref = ctx.lookup(getConfiguration().getString("externalApiPath"));
                        ExternalAPI service = (ExternalAPI) PortableRemoteObject.narrow(ref, ExternalAPI.class);
-                       service.rebuildSolrIndex();
+                       return service.rebuildSolrIndex();
                } catch (ClassCastException e) {
                        throw new JMRuntimeException(e.getMessage());
                } catch (NamingException e) {

diff --git a/src/gr/ebs/gss/mbeans/SolrMBean.java b/src/gr/ebs/gss/mbeans/SolrMBean.java
index e548195..b70aa50 100644 (file)
--- a/src/gr/ebs/gss/mbeans/SolrMBean.java
+++ b/src/gr/ebs/gss/mbeans/SolrMBean.java
@@ -29,5 +29,5 @@ public interface SolrMBean extends ServiceMBean {
     /**
      * Removes the existing index and rebuilds the database from scratch
      */
-    public void rebuildIndex();
+    public String rebuildIndex();
 }

diff --git a/src/gr/ebs/gss/server/ejb/ExternalAPI.java b/src/gr/ebs/gss/server/ejb/ExternalAPI.java
index 9d80371..66806e9 100644 (file)
--- a/src/gr/ebs/gss/server/ejb/ExternalAPI.java
+++ b/src/gr/ebs/gss/server/ejb/ExternalAPI.java
@@ -51,6 +51,7 @@ import javax.ejb.TransactionAttributeType;
 
 import org.apache.commons.configuration.Configuration;
 import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
 
 /**
  * The External API for GSS clients.
@@ -1101,12 +1102,12 @@ public interface ExternalAPI {
        /**
         * It is used by the Solr mbean to rebuild the index.
         */
-       public void rebuildSolrIndex();
+       public String rebuildSolrIndex();
 
        /**
         * It is used by the Solr mbean to refresh the index. It does not delete anything just re-add everything in the index
         */
-       public void refreshSolrIndex();
+       public String refreshSolrIndex();
        
        /**
         * Creates a new file with the specified owner, parent folder and name. The
@@ -1273,8 +1274,9 @@ public interface ExternalAPI {
 
        /**
         * Posts the file specified by id to solr indexing server
-        * 
+        *
+     * @param solr
         * @param id
         */
-       public void postFileToSolr(Long id);
+       public void postFileToSolr(CommonsHttpSolrServer solr, Long id);
 }

diff --git a/src/gr/ebs/gss/server/ejb/ExternalAPIBean.java b/src/gr/ebs/gss/server/ejb/ExternalAPIBean.java
index 20da756..6014525 100644 (file)
--- a/src/gr/ebs/gss/server/ejb/ExternalAPIBean.java
+++ b/src/gr/ebs/gss/server/ejb/ExternalAPIBean.java
@@ -89,7 +89,6 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
-import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
 import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocument;
@@ -848,7 +847,6 @@ public class ExternalAPIBean implements ExternalAPI, ExternalAPIRemote {
         * Retrieve a file for the specified user that has the specified name and
         * its parent folder has id equal to folderId.
         *
-        * @param userId the ID of the current user
         * @param folderId the ID of the parent folder
         * @param name the name of the requested file
         * @return the file found
@@ -2067,18 +2065,29 @@ public class ExternalAPIBean implements ExternalAPI, ExternalAPIRemote {
        }
 
        @Override
-       public void rebuildSolrIndex() {
+    @TransactionAttribute(TransactionAttributeType.NEVER)
+       public String rebuildSolrIndex() {
                try {
-                       CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
+            CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
                        solr.deleteByQuery("*:*");
                        solr.commit();
-                       
+            logger.info("Deleted everything in solr");
+
                        List<Long> fileIds = dao.getAllFileIds();
+            logger.info("Total of " + fileIds.size() + " will be indexed");
+            int i = 0;
                        for (Long id : fileIds) {
-                               postFileToSolr(id);
+                               postFileToSolr(solr, id);
+                i++;
+                if (i % 100 == 0) {
+                    solr.commit();
+                    logger.info("Sent commit to solr at file " + i);
+                }
                        }
                        solr.optimize();
                        solr.commit();
+            logger.info("Finished indexing of " + i + " files");
+            return "Finished indexing of " + i + " files";
                } catch (IOException e) {
                        throw new EJBException(e);
                } catch (SolrServerException e) {
@@ -2087,16 +2096,26 @@ public class ExternalAPIBean implements ExternalAPI, ExternalAPIRemote {
        }
 
        @Override
-       public void refreshSolrIndex() {
+    @TransactionAttribute(TransactionAttributeType.NEVER)
+       public String refreshSolrIndex() {
                try {
                        CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
                        
                        List<Long> fileIds = dao.getAllFileIds();
+            logger.info("Total of " + fileIds.size() + " will be indexed");
+            int i = 0;
                        for (Long id : fileIds) {
-                               postFileToSolr(id);
+                               postFileToSolr(solr, id);
+                i++;
                        }
+            if (i % 100 == 0) {
+                solr.commit();
+                logger.debug("Sent commit to solr at file " + i);
+            }
                        solr.optimize();
                        solr.commit();
+            logger.info("Finished indexing of " + i + " files");
+            return "Finished indexing of " + i + " files";
                } catch (IOException e) {
                        throw new EJBException(e);
                } catch (SolrServerException e) {
@@ -2253,7 +2272,6 @@ public class ExternalAPIBean implements ExternalAPI, ExternalAPIRemote {
         * @param filePath the uploaded file full path
         * @param header the file header that will be associated with the new body
         * @param auditInfo the audit info
-        * @param owner the owner of the file
         * @throws FileNotFoundException
         * @throws QuotaExceededException
         * @throws ObjectNotFoundException if the owner was not found
@@ -2570,9 +2588,9 @@ public class ExternalAPIBean implements ExternalAPI, ExternalAPIRemote {
        }
 
        @Override
-       public void postFileToSolr(Long id) {
+       public void postFileToSolr(CommonsHttpSolrServer solr, Long id) {
                try {
-                       FileHeader file = dao.getEntityById(FileHeader.class, id);
+                       FileHeader file = dao.getFileForIndexing(id);
                        FileBody body = file.getCurrentBody();
                        String mime = body.getMimeType();
                        boolean multipart = true;
@@ -2586,34 +2604,34 @@ public class ExternalAPIBean implements ExternalAPI, ExternalAPIRemote {
                                multipart = false;
 
                        if (!multipart)
-                               sendMetaDataOnly(getConfiguration().getString("solr.url"), file);
+                               sendMetaDataOnly(solr, file);
                        else {
-                               CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
-                               ContentStreamUpdateRequest solrRequest = new ContentStreamUpdateRequest(getConfiguration().getString("solr.rich.update.path"));
+                ContentStreamUpdateRequest solrRequest = new ContentStreamUpdateRequest(getConfiguration().getString("solr.rich.update.path"));
                                solrRequest.setParam("literal.id", file.getId().toString());
                                solrRequest.setParam("literal.name", file.getName());
                                for (FileTag t : file.getFileTags()) {
                                        solrRequest.getParams().add("literal.tag", t.getTag());
                                }
-                               solrRequest.addFile(new File(body.getStoredFilePath()));
-                               solrRequest.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
+                File fsFile = new File(body.getStoredFilePath());
+                               solrRequest.addFile(fsFile);
+//                             solrRequest.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
                                try {
                                        solr.request(solrRequest);
                                }
                                catch (SolrException e) {
                                        logger.warn("File " + id + " failed with " + e.getLocalizedMessage() + ". Retrying without the file");
                                        //Let 's try without the file
-                                       sendMetaDataOnly(getConfiguration().getString("solr.url"), file);
+                                       sendMetaDataOnly(solr, file);
                                }
                                catch (NullPointerException e) {
                                        logger.warn("File " + id + " failed with " + e.getLocalizedMessage() + ". Retrying without the file");
                                        //Let 's try without the file
-                                       sendMetaDataOnly(getConfiguration().getString("solr.url"), file);
+                                       sendMetaDataOnly(solr, file);
                                }
                                catch (SolrServerException e) {
                                        logger.warn("File " + id + " failed with " + e.getLocalizedMessage() + ". Retrying without the file");
                                        //Let 's try without the file
-                                       sendMetaDataOnly(getConfiguration().getString("solr.url"), file);
+                                       sendMetaDataOnly(solr, file);
                                }
                        }
                } catch (MalformedURLException e) {
@@ -2627,8 +2645,7 @@ public class ExternalAPIBean implements ExternalAPI, ExternalAPIRemote {
                }
        }
 
-       private void sendMetaDataOnly(String solrUrl, FileHeader file) throws SolrServerException, IOException {
-               CommonsHttpSolrServer solr = new CommonsHttpSolrServer(solrUrl);
+       private void sendMetaDataOnly(CommonsHttpSolrServer solr, FileHeader file) throws SolrServerException, IOException {
                SolrInputDocument solrDoc = new SolrInputDocument();
                solrDoc.addField("id", file.getId().toString());
                solrDoc.addField("name", file.getName());
@@ -2636,7 +2653,6 @@ public class ExternalAPIBean implements ExternalAPI, ExternalAPIRemote {
                        solrDoc.addField("tag", t.getTag());
                }
                solr.add(solrDoc);
-               solr.commit();
        }
 
        private String tokenizeFilename(String filename){

diff --git a/src/gr/ebs/gss/server/ejb/ExternalAPIRemote.java b/src/gr/ebs/gss/server/ejb/ExternalAPIRemote.java
index b7eb763..db137e7 100644 (file)
--- a/src/gr/ebs/gss/server/ejb/ExternalAPIRemote.java
+++ b/src/gr/ebs/gss/server/ejb/ExternalAPIRemote.java
@@ -705,7 +705,7 @@ public interface ExternalAPIRemote {
        /**
         * It is used by the Solr mbean to rebuild the index.
         */
-       public void rebuildSolrIndex();
+       public String rebuildSolrIndex();
 
        /**
         * Search the system for a user with the specified email address.

diff --git a/src/gr/ebs/gss/server/ejb/GSSDAO.java b/src/gr/ebs/gss/server/ejb/GSSDAO.java
index 3127be9..53583a3 100644 (file)
--- a/src/gr/ebs/gss/server/ejb/GSSDAO.java
+++ b/src/gr/ebs/gss/server/ejb/GSSDAO.java
@@ -463,4 +463,12 @@ public interface GSSDAO {
         */
        public List<FileHeader> getFilesPermittedForGroup(Long userId, Long groupId) throws ObjectNotFoundException;
 
+    /**
+     * Gets a file with tags initialized, cause indexing does not always run within a transaction (e.g. during rebuild)
+     * 
+     * @param id
+     * @return
+     * @throws ObjectNotFoundException
+     */
+    public FileHeader getFileForIndexing(Long id) throws ObjectNotFoundException;
 }

diff --git a/src/gr/ebs/gss/server/ejb/GSSDAOBean.java b/src/gr/ebs/gss/server/ejb/GSSDAOBean.java
index 545d0bf..c0ac05c 100644 (file)
--- a/src/gr/ebs/gss/server/ejb/GSSDAOBean.java
+++ b/src/gr/ebs/gss/server/ejb/GSSDAOBean.java
@@ -697,4 +697,11 @@ public class GSSDAOBean implements GSSDAO {
                                        "where f.owner.id=:userId and f.deleted = false and p.group.id=:groupId ").
                                        setParameter("userId", userId).setParameter("groupId", groupId).getResultList();
        }
+
+    @Override
+    public FileHeader getFileForIndexing(Long id) throws ObjectNotFoundException {
+        FileHeader h = getEntityById(FileHeader.class, id);
+        h.getFileTags().size();
+        return h;
+    }
 }

diff --git a/src/gr/ebs/gss/server/ejb/indexer/IndexerMDBean.java b/src/gr/ebs/gss/server/ejb/indexer/IndexerMDBean.java
index 2cca442..d525f38 100644 (file)
--- a/src/gr/ebs/gss/server/ejb/indexer/IndexerMDBean.java
+++ b/src/gr/ebs/gss/server/ejb/indexer/IndexerMDBean.java
@@ -97,10 +97,13 @@ public class IndexerMDBean implements MessageListener {
                        id = (Long) map.getObject("id");
                        boolean delete = map.getBoolean("delete");
                        Configuration config = GSSConfigurationFactory.getConfiguration();
-                       if (delete) {
-                               sendDelete(config.getString("solr.url"), id);
+            CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
+            if (delete) {
+                               sendDelete(solr, id);
+                solr.commit();
                        } else {
-                               service.postFileToSolr(id);
+                               service.postFileToSolr(solr, id);
+                solr.commit();
                        }       
                }
                catch (JMSException e) {
@@ -118,16 +121,12 @@ public class IndexerMDBean implements MessageListener {
        /**
         * Sends a delete command to solr. The id is the Long id of the indexed document
         * 
-        * @param solrUrl
+        * @param solr
         * @param id
         * @throws SolrServerException
         * @throws IOException
         */
-       private void sendDelete(String solrUrl, Long id)        throws SolrServerException, IOException {
-               CommonsHttpSolrServer solr = new CommonsHttpSolrServer(solrUrl);
+       private void sendDelete(CommonsHttpSolrServer solr, Long id)    throws SolrServerException, IOException {
                solr.deleteById(id.toString());
-               solr.commit();
        }
-
-
 }