Revision 376d0ebf
| b/src/gr/ebs/gss/mbeans/Solr.java | ||
|---|---|---|
| 37 | 37 |
public class Solr extends ServiceMBeanSupport implements SolrMBean {
|
| 38 | 38 |
|
| 39 | 39 |
@Override |
| 40 |
public void rebuildIndex() {
|
|
| 40 |
public String rebuildIndex() {
|
|
| 41 | 41 |
try {
|
| 42 | 42 |
InitialContext ctx = new InitialContext(); |
| 43 | 43 |
Object ref = ctx.lookup(getConfiguration().getString("externalApiPath"));
|
| 44 | 44 |
ExternalAPI service = (ExternalAPI) PortableRemoteObject.narrow(ref, ExternalAPI.class); |
| 45 |
service.rebuildSolrIndex(); |
|
| 45 |
return service.rebuildSolrIndex();
|
|
| 46 | 46 |
} catch (ClassCastException e) {
|
| 47 | 47 |
throw new JMRuntimeException(e.getMessage()); |
| 48 | 48 |
} catch (NamingException e) {
|
| b/src/gr/ebs/gss/mbeans/SolrMBean.java | ||
|---|---|---|
| 29 | 29 |
/** |
| 30 | 30 |
* Removes the existing index and rebuilds the database from scratch |
| 31 | 31 |
*/ |
| 32 |
public void rebuildIndex();
|
|
| 32 |
public String rebuildIndex();
|
|
| 33 | 33 |
} |
| b/src/gr/ebs/gss/server/ejb/ExternalAPI.java | ||
|---|---|---|
| 51 | 51 |
|
| 52 | 52 |
import org.apache.commons.configuration.Configuration; |
| 53 | 53 |
import org.apache.solr.client.solrj.SolrServerException; |
| 54 |
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; |
|
| 54 | 55 |
|
| 55 | 56 |
/** |
| 56 | 57 |
* The External API for GSS clients. |
| ... | ... | |
| 1101 | 1102 |
/** |
| 1102 | 1103 |
* It is used by the Solr mbean to rebuild the index. |
| 1103 | 1104 |
*/ |
| 1104 |
public void rebuildSolrIndex();
|
|
| 1105 |
public String rebuildSolrIndex();
|
|
| 1105 | 1106 |
|
| 1106 | 1107 |
/** |
| 1107 | 1108 |
* It is used by the Solr mbean to refresh the index. It does not delete anything just re-add everything in the index |
| 1108 | 1109 |
*/ |
| 1109 |
public void refreshSolrIndex();
|
|
| 1110 |
public String refreshSolrIndex();
|
|
| 1110 | 1111 |
|
| 1111 | 1112 |
/** |
| 1112 | 1113 |
* Creates a new file with the specified owner, parent folder and name. The |
| ... | ... | |
| 1273 | 1274 |
|
| 1274 | 1275 |
/** |
| 1275 | 1276 |
* Posts the file specified by id to solr indexing server |
| 1276 |
* |
|
| 1277 |
* |
|
| 1278 |
* @param solr |
|
| 1277 | 1279 |
* @param id |
| 1278 | 1280 |
*/ |
| 1279 |
public void postFileToSolr(Long id); |
|
| 1281 |
public void postFileToSolr(CommonsHttpSolrServer solr, Long id);
|
|
| 1280 | 1282 |
} |
| b/src/gr/ebs/gss/server/ejb/ExternalAPIBean.java | ||
|---|---|---|
| 89 | 89 |
import org.apache.solr.client.solrj.SolrQuery; |
| 90 | 90 |
import org.apache.solr.client.solrj.SolrServerException; |
| 91 | 91 |
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; |
| 92 |
import org.apache.solr.client.solrj.request.AbstractUpdateRequest; |
|
| 93 | 92 |
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; |
| 94 | 93 |
import org.apache.solr.client.solrj.response.QueryResponse; |
| 95 | 94 |
import org.apache.solr.common.SolrDocument; |
| ... | ... | |
| 848 | 847 |
* Retrieve a file for the specified user that has the specified name and |
| 849 | 848 |
* its parent folder has id equal to folderId. |
| 850 | 849 |
* |
| 851 |
* @param userId the ID of the current user |
|
| 852 | 850 |
* @param folderId the ID of the parent folder |
| 853 | 851 |
* @param name the name of the requested file |
| 854 | 852 |
* @return the file found |
| ... | ... | |
| 2067 | 2065 |
} |
| 2068 | 2066 |
|
| 2069 | 2067 |
@Override |
| 2070 |
public void rebuildSolrIndex() {
|
|
| 2068 |
@TransactionAttribute(TransactionAttributeType.NEVER) |
|
| 2069 |
public String rebuildSolrIndex() {
|
|
| 2071 | 2070 |
try {
|
| 2072 |
CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
|
|
| 2071 |
CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
|
|
| 2073 | 2072 |
solr.deleteByQuery("*:*");
|
| 2074 | 2073 |
solr.commit(); |
| 2075 |
|
|
| 2074 |
logger.info("Deleted everything in solr");
|
|
| 2075 |
|
|
| 2076 | 2076 |
List<Long> fileIds = dao.getAllFileIds(); |
| 2077 |
logger.info("Total of " + fileIds.size() + " will be indexed");
|
|
| 2078 |
int i = 0; |
|
| 2077 | 2079 |
for (Long id : fileIds) {
|
| 2078 |
postFileToSolr(id); |
|
| 2080 |
postFileToSolr(solr, id); |
|
| 2081 |
i++; |
|
| 2082 |
if (i % 100 == 0) {
|
|
| 2083 |
solr.commit(); |
|
| 2084 |
logger.info("Sent commit to solr at file " + i);
|
|
| 2085 |
} |
|
| 2079 | 2086 |
} |
| 2080 | 2087 |
solr.optimize(); |
| 2081 | 2088 |
solr.commit(); |
| 2089 |
logger.info("Finished indexing of " + i + " files");
|
|
| 2090 |
return "Finished indexing of " + i + " files"; |
|
| 2082 | 2091 |
} catch (IOException e) {
|
| 2083 | 2092 |
throw new EJBException(e); |
| 2084 | 2093 |
} catch (SolrServerException e) {
|
| ... | ... | |
| 2087 | 2096 |
} |
| 2088 | 2097 |
|
| 2089 | 2098 |
@Override |
| 2090 |
public void refreshSolrIndex() {
|
|
| 2099 |
@TransactionAttribute(TransactionAttributeType.NEVER) |
|
| 2100 |
public String refreshSolrIndex() {
|
|
| 2091 | 2101 |
try {
|
| 2092 | 2102 |
CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
|
| 2093 | 2103 |
|
| 2094 | 2104 |
List<Long> fileIds = dao.getAllFileIds(); |
| 2105 |
logger.info("Total of " + fileIds.size() + " will be indexed");
|
|
| 2106 |
int i = 0; |
|
| 2095 | 2107 |
for (Long id : fileIds) {
|
| 2096 |
postFileToSolr(id); |
|
| 2108 |
postFileToSolr(solr, id); |
|
| 2109 |
i++; |
|
| 2097 | 2110 |
} |
| 2111 |
if (i % 100 == 0) {
|
|
| 2112 |
solr.commit(); |
|
| 2113 |
logger.debug("Sent commit to solr at file " + i);
|
|
| 2114 |
} |
|
| 2098 | 2115 |
solr.optimize(); |
| 2099 | 2116 |
solr.commit(); |
| 2117 |
logger.info("Finished indexing of " + i + " files");
|
|
| 2118 |
return "Finished indexing of " + i + " files"; |
|
| 2100 | 2119 |
} catch (IOException e) {
|
| 2101 | 2120 |
throw new EJBException(e); |
| 2102 | 2121 |
} catch (SolrServerException e) {
|
| ... | ... | |
| 2253 | 2272 |
* @param filePath the uploaded file full path |
| 2254 | 2273 |
* @param header the file header that will be associated with the new body |
| 2255 | 2274 |
* @param auditInfo the audit info |
| 2256 |
* @param owner the owner of the file |
|
| 2257 | 2275 |
* @throws FileNotFoundException |
| 2258 | 2276 |
* @throws QuotaExceededException |
| 2259 | 2277 |
* @throws ObjectNotFoundException if the owner was not found |
| ... | ... | |
| 2570 | 2588 |
} |
| 2571 | 2589 |
|
| 2572 | 2590 |
@Override |
| 2573 |
public void postFileToSolr(Long id) {
|
|
| 2591 |
public void postFileToSolr(CommonsHttpSolrServer solr, Long id) {
|
|
| 2574 | 2592 |
try {
|
| 2575 |
FileHeader file = dao.getEntityById(FileHeader.class, id);
|
|
| 2593 |
FileHeader file = dao.getFileForIndexing(id);
|
|
| 2576 | 2594 |
FileBody body = file.getCurrentBody(); |
| 2577 | 2595 |
String mime = body.getMimeType(); |
| 2578 | 2596 |
boolean multipart = true; |
| ... | ... | |
| 2586 | 2604 |
multipart = false; |
| 2587 | 2605 |
|
| 2588 | 2606 |
if (!multipart) |
| 2589 |
sendMetaDataOnly(getConfiguration().getString("solr.url"), file);
|
|
| 2607 |
sendMetaDataOnly(solr, file);
|
|
| 2590 | 2608 |
else {
|
| 2591 |
CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
|
|
| 2592 |
ContentStreamUpdateRequest solrRequest = new ContentStreamUpdateRequest(getConfiguration().getString("solr.rich.update.path"));
|
|
| 2609 |
ContentStreamUpdateRequest solrRequest = new ContentStreamUpdateRequest(getConfiguration().getString("solr.rich.update.path"));
|
|
| 2593 | 2610 |
solrRequest.setParam("literal.id", file.getId().toString());
|
| 2594 | 2611 |
solrRequest.setParam("literal.name", file.getName());
|
| 2595 | 2612 |
for (FileTag t : file.getFileTags()) {
|
| 2596 | 2613 |
solrRequest.getParams().add("literal.tag", t.getTag());
|
| 2597 | 2614 |
} |
| 2598 |
solrRequest.addFile(new File(body.getStoredFilePath())); |
|
| 2599 |
solrRequest.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); |
|
| 2615 |
File fsFile = new File(body.getStoredFilePath()); |
|
| 2616 |
solrRequest.addFile(fsFile); |
|
| 2617 |
// solrRequest.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); |
|
| 2600 | 2618 |
try {
|
| 2601 | 2619 |
solr.request(solrRequest); |
| 2602 | 2620 |
} |
| 2603 | 2621 |
catch (SolrException e) {
|
| 2604 | 2622 |
logger.warn("File " + id + " failed with " + e.getLocalizedMessage() + ". Retrying without the file");
|
| 2605 | 2623 |
//Let 's try without the file |
| 2606 |
sendMetaDataOnly(getConfiguration().getString("solr.url"), file);
|
|
| 2624 |
sendMetaDataOnly(solr, file);
|
|
| 2607 | 2625 |
} |
| 2608 | 2626 |
catch (NullPointerException e) {
|
| 2609 | 2627 |
logger.warn("File " + id + " failed with " + e.getLocalizedMessage() + ". Retrying without the file");
|
| 2610 | 2628 |
//Let 's try without the file |
| 2611 |
sendMetaDataOnly(getConfiguration().getString("solr.url"), file);
|
|
| 2629 |
sendMetaDataOnly(solr, file);
|
|
| 2612 | 2630 |
} |
| 2613 | 2631 |
catch (SolrServerException e) {
|
| 2614 | 2632 |
logger.warn("File " + id + " failed with " + e.getLocalizedMessage() + ". Retrying without the file");
|
| 2615 | 2633 |
//Let 's try without the file |
| 2616 |
sendMetaDataOnly(getConfiguration().getString("solr.url"), file);
|
|
| 2634 |
sendMetaDataOnly(solr, file);
|
|
| 2617 | 2635 |
} |
| 2618 | 2636 |
} |
| 2619 | 2637 |
} catch (MalformedURLException e) {
|
| ... | ... | |
| 2627 | 2645 |
} |
| 2628 | 2646 |
} |
| 2629 | 2647 |
|
| 2630 |
private void sendMetaDataOnly(String solrUrl, FileHeader file) throws SolrServerException, IOException {
|
|
| 2631 |
CommonsHttpSolrServer solr = new CommonsHttpSolrServer(solrUrl); |
|
| 2648 |
private void sendMetaDataOnly(CommonsHttpSolrServer solr, FileHeader file) throws SolrServerException, IOException {
|
|
| 2632 | 2649 |
SolrInputDocument solrDoc = new SolrInputDocument(); |
| 2633 | 2650 |
solrDoc.addField("id", file.getId().toString());
|
| 2634 | 2651 |
solrDoc.addField("name", file.getName());
|
| ... | ... | |
| 2636 | 2653 |
solrDoc.addField("tag", t.getTag());
|
| 2637 | 2654 |
} |
| 2638 | 2655 |
solr.add(solrDoc); |
| 2639 |
solr.commit(); |
|
| 2640 | 2656 |
} |
| 2641 | 2657 |
|
| 2642 | 2658 |
private String tokenizeFilename(String filename){
|
| b/src/gr/ebs/gss/server/ejb/ExternalAPIRemote.java | ||
|---|---|---|
| 705 | 705 |
/** |
| 706 | 706 |
* It is used by the Solr mbean to rebuild the index. |
| 707 | 707 |
*/ |
| 708 |
public void rebuildSolrIndex();
|
|
| 708 |
public String rebuildSolrIndex();
|
|
| 709 | 709 |
|
| 710 | 710 |
/** |
| 711 | 711 |
* Search the system for a user with the specified email address. |
| b/src/gr/ebs/gss/server/ejb/GSSDAO.java | ||
|---|---|---|
| 463 | 463 |
*/ |
| 464 | 464 |
public List<FileHeader> getFilesPermittedForGroup(Long userId, Long groupId) throws ObjectNotFoundException; |
| 465 | 465 |
|
| 466 |
/** |
|
| 467 |
* Gets a file with tags initialized, cause indexing does not always run within a transaction (e.g. during rebuild) |
|
| 468 |
* |
|
| 469 |
* @param id |
|
| 470 |
* @return |
|
| 471 |
* @throws ObjectNotFoundException |
|
| 472 |
*/ |
|
| 473 |
public FileHeader getFileForIndexing(Long id) throws ObjectNotFoundException; |
|
| 466 | 474 |
} |
| b/src/gr/ebs/gss/server/ejb/GSSDAOBean.java | ||
|---|---|---|
| 697 | 697 |
"where f.owner.id=:userId and f.deleted = false and p.group.id=:groupId "). |
| 698 | 698 |
setParameter("userId", userId).setParameter("groupId", groupId).getResultList();
|
| 699 | 699 |
} |
| 700 |
|
|
| 701 |
@Override |
|
| 702 |
public FileHeader getFileForIndexing(Long id) throws ObjectNotFoundException {
|
|
| 703 |
FileHeader h = getEntityById(FileHeader.class, id); |
|
| 704 |
h.getFileTags().size(); |
|
| 705 |
return h; |
|
| 706 |
} |
|
| 700 | 707 |
} |
| b/src/gr/ebs/gss/server/ejb/indexer/IndexerMDBean.java | ||
|---|---|---|
| 97 | 97 |
id = (Long) map.getObject("id");
|
| 98 | 98 |
boolean delete = map.getBoolean("delete");
|
| 99 | 99 |
Configuration config = GSSConfigurationFactory.getConfiguration(); |
| 100 |
if (delete) {
|
|
| 101 |
sendDelete(config.getString("solr.url"), id);
|
|
| 100 |
CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
|
|
| 101 |
if (delete) {
|
|
| 102 |
sendDelete(solr, id); |
|
| 103 |
solr.commit(); |
|
| 102 | 104 |
} else {
|
| 103 |
service.postFileToSolr(id); |
|
| 105 |
service.postFileToSolr(solr, id); |
|
| 106 |
solr.commit(); |
|
| 104 | 107 |
} |
| 105 | 108 |
} |
| 106 | 109 |
catch (JMSException e) {
|
| ... | ... | |
| 118 | 121 |
/** |
| 119 | 122 |
* Sends a delete command to solr. The id is the Long id of the indexed document |
| 120 | 123 |
* |
| 121 |
* @param solrUrl
|
|
| 124 |
* @param solr |
|
| 122 | 125 |
* @param id |
| 123 | 126 |
* @throws SolrServerException |
| 124 | 127 |
* @throws IOException |
| 125 | 128 |
*/ |
| 126 |
private void sendDelete(String solrUrl, Long id) throws SolrServerException, IOException {
|
|
| 127 |
CommonsHttpSolrServer solr = new CommonsHttpSolrServer(solrUrl); |
|
| 129 |
private void sendDelete(CommonsHttpSolrServer solr, Long id) throws SolrServerException, IOException {
|
|
| 128 | 130 |
solr.deleteById(id.toString()); |
| 129 |
solr.commit(); |
|
| 130 | 131 |
} |
| 131 |
|
|
| 132 |
|
|
| 133 | 132 |
} |
Also available in: Unified diff