import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
+import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import javax.naming.InitialContext;
import javax.naming.NamingException;
import javax.persistence.PersistenceException;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerConfigurationException;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.NameValuePair;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.httpclient.methods.StringRequestEntity;
+
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
import org.hibernate.exception.ConstraintViolationException;
-import org.w3c.dom.DOMException;
-import org.w3c.dom.Document;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
import com.novell.ldap.LDAPAttribute;
import com.novell.ldap.LDAPAttributeSet;
}
}
+ private Long getRootFolderId(Long userId) throws ObjectNotFoundException {
+ if (userId == null)
+ throw new ObjectNotFoundException("No user specified");
+ return dao.getRootFolderId(userId);
+ }
+
@Override
public FolderDTO getRootFolder(Long userId) throws ObjectNotFoundException {
if (userId == null)
folder.getAuditInfo().setModifiedBy(user);
dao.update(folder);
touchParentFolders(folder, user, new Date());
+ // Re-index the folder contents if it was modified.
+ if ((permissions != null && !permissions.isEmpty()) || readForAll != null) {
+ indexFolder(folder);
+ }
+
return folder.getDTO();
}
+ private void indexFolder(Folder folder) {
+ for (FileHeader fh : folder.getFiles())
+ indexFile(fh.getId(), false);
+ for (Folder f : folder.getSubfolders())
+ indexFolder(f);
+ }
+
@Override
public void createGroup(final Long userId, final String name) throws ObjectNotFoundException, DuplicateNameException {
// Validate.
touchParentFolders(parent, user, new Date());
// Re-index the file if it was modified.
- if (name != null || tagSet != null)
+ if (name != null || tagSet != null || (permissions != null && !permissions.isEmpty()) || readForAll != null)
indexFile(fileId, false);
}
return getRootFolder(owner.getId());
// Store the last element, since it requires special handling.
String lastElement = pathElements.remove(pathElements.size() - 1);
- FolderDTO cursor = getRootFolder(owner.getId());
+
+ Folder cursor = null;
+ Long rootFolderId = getRootFolderId(owner.getId());
// Traverse and verify the specified folder path.
for (String pathElement : pathElements) {
- cursor = getFolder(cursor.getId(), pathElement);
+ cursor = getFolder(cursor==null ? rootFolderId : cursor.getId(), pathElement);
if (cursor.isDeleted())
throw new ObjectNotFoundException("Folder " + cursor.getPath() + " not found");
}
// Use the lastElement to retrieve the actual resource.
Object resource = null;
try {
- FileHeaderDTO file = getFile(cursor.getId(), lastElement);
+ FileHeaderDTO file = getFile(cursor==null ? rootFolderId : cursor.getId(), lastElement);
if (ignoreDeleted && file.isDeleted())
throw new ObjectNotFoundException("Resource not found");
resource = file;
} catch (ObjectNotFoundException e) {
// Perhaps the requested resource is not a file, so
// check for folders as well.
- FolderDTO folder = getFolder(cursor.getId(), lastElement);
+ FolderDTO folder = getFolder(cursor==null ? rootFolderId : cursor.getId(), lastElement).getDTO();
if (ignoreDeleted && folder.isDeleted())
throw new ObjectNotFoundException("Resource not found");
resource = folder;
* Retrieve a file for the specified user that has the specified name and
* its parent folder has id equal to folderId.
*
- * @param userId the ID of the current user
* @param folderId the ID of the parent folder
* @param name the name of the requested file
* @return the file found
* found, with the exception message mentioning the precise
* problem
*/
- private FolderDTO getFolder(Long parentId, String name) throws ObjectNotFoundException {
+ private Folder getFolder(Long parentId, String name) throws ObjectNotFoundException {
if (parentId == null)
throw new ObjectNotFoundException("No parent folder specified");
if (StringUtils.isEmpty(name))
throw new ObjectNotFoundException("No folder specified");
Folder folder = dao.getFolder(parentId, name);
- return folder.getDTO();
+ return folder;
}
private FileHeaderDTO updateFileContents(Long userId, Long fileId, String mimeType, InputStream resourceInputStream) throws ObjectNotFoundException, GSSIOException, InsufficientPermissionsException, QuotaExceededException {
}
@Override
- public List<FileHeaderDTO> searchFiles(Long userId, String query) throws ObjectNotFoundException {
+ public List<FileHeader> searchFiles(Long userId, String query) throws ObjectNotFoundException {
+ long startTime = System.currentTimeMillis();
if (userId == null)
throw new ObjectNotFoundException("No user specified");
User user = getUser(userId);
if (query == null)
throw new ObjectNotFoundException("No query specified");
List<FileHeader> files = search(user.getId(), query);
- List<FileHeaderDTO> res = new ArrayList<FileHeaderDTO>();
- for(FileHeader f : files)
- res.add(f.getDTO());
- return res;
+
+ long stopTime = System.currentTimeMillis();
+ logger.info("Total time: " + (stopTime - startTime));
+ return files;
}
/**
* Performs the actuals search on the solr server and returns the results
*
- * We have to use the dismax query type (instead of the
- * standard) because it allows for search time field boosting. This is because we can't use indexing
- * time field boosting due to the patched rich indexing API that does not allow it
- *
* @param userId
* @param query
* @return a List of FileHeader objects
*/
private List<FileHeader> search(Long userId, String query) {
+ final int maxRows = 100;
+ List<FileHeader> result = new ArrayList<FileHeader>();
try {
- HttpClient httpClient = new HttpClient();
-
- GetMethod method = new GetMethod(getConfiguration().getString("solrSelectUrl"));
- NameValuePair[] params = {new NameValuePair("qt", "dismax"),
- new NameValuePair("q", query),
- new NameValuePair("sort", "score desc"),
- new NameValuePair("indent", "on")};
- method.setQueryString(params);
- int retryCount = 0;
- int statusCode = 0;
- String response = null;
- do {
- statusCode = httpClient.executeMethod(method);
- logger.debug("HTTP status: " + statusCode);
- response = method.getResponseBodyAsString();
- logger.debug(response);
- retryCount++;
- if (statusCode != 200 && retryCount < 3)
- try {
- Thread.sleep(3000); //Give Solr a little time to be available
- } catch (InterruptedException e) {
- }
- } while (statusCode != 200 && retryCount < 3);
- if (statusCode != 200)
- throw new EJBException("Search query return error:\n" + response);
-
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- DocumentBuilder db = dbf.newDocumentBuilder();
- Document doc = db.parse(method.getResponseBodyAsStream());
- method.releaseConnection();
-
- Node root = doc.getElementsByTagName("response").item(0);
- Node lst = root.getFirstChild().getNextSibling();
- Node status = lst.getFirstChild().getNextSibling();
- if (status.getAttributes().getNamedItem("name").getNodeValue().equals("status") &&
- status.getTextContent().equals("0")) {
- List<FileHeader> fileResult = new ArrayList<FileHeader>();
- Node result = lst.getNextSibling().getNextSibling();
- NodeList docs = result.getChildNodes();
- User user = getUser(userId);
- for (int i=1; i<docs.getLength(); i=i+2) {
- Node d = docs.item(i);
- NodeList docData = d.getChildNodes();
- for (int j=1; j<docData.getLength(); j=j+2) {
- Node dd = docData.item(j);
- if (dd.getAttributes().item(0).getNodeName().equals("name") &&
- dd.getAttributes().item(0).getNodeValue().equals("id")) {
- Long fileId = Long.valueOf(dd.getTextContent());
- try {
- FileHeader file = dao.getEntityById(FileHeader.class, fileId);
- if (file.hasReadPermission(user)) {
- fileResult.add(file);
- logger.debug("File added " + fileId);
- }
- } catch (ObjectNotFoundException e) {
- logger.warn("Search result not found", e);
- }
- }
- }
+ CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
+ List<Group> groups = dao.getGroupsContainingUser(userId);
+ String constructedQuery = escapeCharacters(normalizeSearchQuery(query)) + " AND (public: true OR ureaders: " + userId;
+ if (!groups.isEmpty()) {
+ constructedQuery += " OR (";
+ for (int i=0; i<groups.size(); i++) {
+ Group g = groups.get(i);
+ constructedQuery += "greaders :" + g.getId();
+ if (i < groups.size() - 1)
+ constructedQuery += " OR ";
+ }
+ constructedQuery += ")";
+ }
+ constructedQuery += ")";
+ SolrQuery solrQuery = new SolrQuery(constructedQuery);
+ solrQuery.setRows(maxRows);
+ long startTime = System.currentTimeMillis();
+ QueryResponse response = solr.query(solrQuery);
+ SolrDocumentList results = response.getResults();
+ if (results.getNumFound() > maxRows) {
+ solrQuery.setRows(Integer.valueOf((int) results.getNumFound()));
+ response = solr.query(solrQuery);
+ results = response.getResults();
+ }
+ long stopTime = System.currentTimeMillis();
+ logger.info("Search time:" + (stopTime - startTime));
+ User user = getUser(userId);
+ startTime = System.currentTimeMillis();
+ for (SolrDocument d : results) {
+ Long id = Long.valueOf((String) d.getFieldValue("id"));
+ try {
+ FileHeader f = dao.getEntityById(FileHeader.class, id);
+ result.add(f);
+ } catch (ObjectNotFoundException e) {
+ logger.warn("Search result id " + id + " cannot be found", e);
}
- return fileResult;
}
- throw new EJBException();
- } catch (HttpException e) {
- throw new EJBException(e);
- } catch (IOException e) {
+ stopTime = System.currentTimeMillis();
+ logger.info("File loads: " + (stopTime - startTime));
+ } catch (MalformedURLException e) {
+ logger.error(e);
throw new EJBException(e);
- } catch (SAXException e) {
- throw new EJBException(e);
- } catch (ParserConfigurationException e) {
+ } catch (SolrServerException e) {
+ logger.error(e);
throw new EJBException(e);
} catch (ObjectNotFoundException e) {
+ logger.error(e);
throw new EJBException(e);
}
+ return result;
}
@Override
}
@Override
- public void rebuildSolrIndex() {
- MessageProducer sender = null;
- Session session = null;
- Connection qConn = null;
+ @TransactionAttribute(TransactionAttributeType.NEVER)
+ public String rebuildSolrIndex() {
try {
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- DocumentBuilder db = dbf.newDocumentBuilder();
- Document doc = db.newDocument();
- Node root = doc.createElement("delete");
- doc.appendChild(root);
- Node queryNode = doc.createElement("query");
- root.appendChild(queryNode);
- queryNode.appendChild(doc.createTextNode("*:*"));
-
- TransformerFactory fact = TransformerFactory.newInstance();
- Transformer trans = fact.newTransformer();
- trans.setOutputProperty(OutputKeys.INDENT, "yes");
- StringWriter sw = new StringWriter();
- StreamResult sr = new StreamResult(sw);
- DOMSource source = new DOMSource(doc);
- trans.transform(source, sr);
- logger.debug(sw.toString());
-
- HttpClient httpClient = new HttpClient();
- PostMethod method = new PostMethod(getConfiguration().getString("solrUpdateUrl"));
- method.setRequestEntity(new StringRequestEntity(sw.toString()));
- int retryCount = 0;
- int statusCode = 0;
- String response = null;
- do {
- statusCode = httpClient.executeMethod(method);
- logger.debug("HTTP status: " + statusCode);
- response = method.getResponseBodyAsString();
- logger.debug(response);
- retryCount++;
- if (statusCode != 200 && retryCount < 3)
- try {
- Thread.sleep(10000); //Give Solr a little time to be available
- } catch (InterruptedException e) {
- }
- } while (statusCode != 200 && retryCount < 3);
- method.releaseConnection();
- if (statusCode != 200)
- throw new EJBException("Cannot clear Solr index. Solr response is:\n" + response);
- List<Long> fileIds = dao.getAllFileIds();
-
- Context jndiCtx = new InitialContext();
- ConnectionFactory factory = (QueueConnectionFactory) jndiCtx.lookup("java:/JmsXA");
- Queue queue = (Queue) jndiCtx.lookup("queue/gss-indexingQueue");
- qConn = factory.createConnection();
- session = qConn.createSession(false, Session.AUTO_ACKNOWLEDGE);
- sender = session.createProducer(queue);
+ CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
+ solr.deleteByQuery("*:*");
+ solr.commit();
+ logger.info("Deleted everything in solr");
+ List<Long> fileIds = dao.getAllFileIds();
+ logger.info("Total of " + fileIds.size() + " will be indexed");
+ int i = 0;
for (Long id : fileIds) {
- MapMessage map = session.createMapMessage();
- map.setObject("id", id);
- map.setBoolean("delete", false);
- sender.send(map);
+ postFileToSolr(solr, id);
+ i++;
+ if (i % 10 == 0) {
+ solr.commit();
+ logger.info("Sent commit to solr at file " + i);
+ }
}
- sendOptimize(httpClient, 0);
- } catch (DOMException e) {
- throw new EJBException(e);
- } catch (TransformerConfigurationException e) {
- throw new EJBException(e);
- } catch (IllegalArgumentException e) {
- throw new EJBException(e);
- } catch (HttpException e) {
- throw new EJBException(e);
- } catch (UnsupportedEncodingException e) {
- throw new EJBException(e);
- } catch (ParserConfigurationException e) {
- throw new EJBException(e);
- } catch (TransformerException e) {
- throw new EJBException(e);
+ solr.optimize();
+ solr.commit();
+ logger.info("Finished indexing of " + i + " files");
+ return "Finished indexing of " + i + " files";
} catch (IOException e) {
throw new EJBException(e);
- } catch (NamingException e) {
- throw new EJBException(e);
- } catch (JMSException e) {
+ } catch (SolrServerException e) {
throw new EJBException(e);
}
- finally {
- try {
- if (sender != null)
- sender.close();
- if (session != null)
- session.close();
- if (qConn != null)
- qConn.close();
- }
- catch (JMSException e) {
- logger.warn(e);
- }
- }
}
- /**
- * Sends a optimize message to the solr server
- *
- * @param httpClient
- * @param retryCount If the commit fails, it is retried three times. This parameter is passed in the recursive
- * calls to stop the recursion
- * @throws UnsupportedEncodingException
- * @throws IOException
- * @throws HttpException
- */
- private void sendOptimize(HttpClient httpClient, int retryCount) throws UnsupportedEncodingException, IOException, HttpException {
- PostMethod method = null;
+ @Override
+ @TransactionAttribute(TransactionAttributeType.NEVER)
+ public String refreshSolrIndex() {
try {
- logger.debug("Optimize retry: " + retryCount);
- method = new PostMethod(getConfiguration().getString("solrUpdateUrl"));
- method.setRequestEntity(new StringRequestEntity("<optimize/>", "text/xml", "iso8859-1"));
- int statusCode = httpClient.executeMethod(method);
- logger.debug("HTTP status: " + statusCode);
- String response = method.getResponseBodyAsString();
- logger.debug(response);
- if (statusCode != 200 && retryCount < 2) {
- try {
- Thread.sleep(10000); //Give Solr a little time to be available
- } catch (InterruptedException e) {
- }
- sendOptimize(httpClient, retryCount + 1);
+ CommonsHttpSolrServer solr = new CommonsHttpSolrServer(getConfiguration().getString("solr.url"));
+
+ List<Long> fileIds = dao.getAllFileIds();
+ logger.info("Total of " + fileIds.size() + " will be indexed");
+ int i = 0;
+ for (Long id : fileIds) {
+ postFileToSolr(solr, id);
+ i++;
}
- }
- finally {
- if (method != null)
- method.releaseConnection();
+ if (i % 10 == 0) {
+ solr.commit();
+ logger.info("Sent commit to solr at file " + i);
+ }
+ solr.optimize();
+ solr.commit();
+ logger.info("Finished indexing of " + i + " files");
+ return "Finished indexing of " + i + " files";
+ } catch (IOException e) {
+ throw new EJBException(e);
+ } catch (SolrServerException e) {
+ throw new EJBException(e);
}
}
* @param filePath the uploaded file full path
* @param header the file header that will be associated with the new body
* @param auditInfo the audit info
- * @param owner the owner of the file
* @throws FileNotFoundException
* @throws QuotaExceededException
* @throws ObjectNotFoundException if the owner was not found
* @throws ObjectNotFoundException
*/
- public List<UserLogin> getUserLogins(Long userId) throws ObjectNotFoundException{
+ public List<UserLogin> getLastUserLogins(Long userId) throws ObjectNotFoundException{
List<UserLogin> userLoginResults = new ArrayList<UserLogin>();
userLoginResults = dao.getLoginsForUser(userId);
if(userLoginResults.size() == 0)
return userLoginResults;
}
+
+ @Override
+ public void postFileToSolr(CommonsHttpSolrServer solr, Long id) {
+ try {
+ FileHeader file = dao.getFileForIndexing(id);
+ FileBody body = file.getCurrentBody();
+ String mime = body.getMimeType();
+ boolean multipart = true;
+ if (!mime.equals("application/pdf")
+ && !mime.equals("text/plain")
+ && !mime.equals("text/html")
+ && !mime.endsWith("msword")
+ && !mime.endsWith("ms-excel")
+ && !mime.endsWith("powerpoint")
+ || (body.getFileSize() > getConfiguration().getLong("solrDocumentUploadLimitInKB") * 1024))
+ multipart = false;
+
+ if (!multipart)
+ sendMetaDataOnly(solr, file);
+ else {
+ ContentStreamUpdateRequest solrRequest = new ContentStreamUpdateRequest(getConfiguration().getString("solr.rich.update.path"));
+ solrRequest.setParam("literal.id", file.getId().toString());
+ solrRequest.setParam("literal.name", file.getName());
+ for (FileTag t : file.getFileTags()) {
+ solrRequest.getParams().add("literal.tag", t.getTag());
+ }
+ for (Permission p : file.getPermissions()) {
+ if (p.getRead()) {
+ if (p.getUser() != null)
+ solrRequest.setParam("literal.ureaders", p.getUser().getId().toString());
+ else if (p.getGroup() != null)
+ solrRequest.setParam("literal.greaders", p.getGroup().getId().toString());
+ }
+ }
+ solrRequest.setParam("literal.owner", file.getOwner().getId().toString());
+ solrRequest.setParam("literal.public", String.valueOf(file.isReadForAll()));
+ File fsFile = new File(body.getStoredFilePath());
+ solrRequest.addFile(fsFile);
+ try {
+ solr.request(solrRequest);
+ }
+ catch (SolrException e) {
+ logger.warn("File " + id + " failed with SolrException: " + e.getLocalizedMessage() + ". Retrying without the file");
+ //Let 's try without the file
+ sendMetaDataOnly(solr, file);
+ }
+ catch (NullPointerException e) {
+ logger.warn("File " + id + " failed with NullPointerException: " + e.getLocalizedMessage() + ". Retrying without the file");
+ //Let 's try without the file
+ sendMetaDataOnly(solr, file);
+ }
+ catch (SolrServerException e) {
+ logger.warn("File " + id + " failed with SolrServerException: " + e.getLocalizedMessage() + ". Retrying without the file");
+ //Let 's try without the file
+ sendMetaDataOnly(solr, file);
+ }
+ }
+ } catch (MalformedURLException e) {
+ throw new EJBException(e);
+ } catch (ObjectNotFoundException e) {
+ logger.error("Indexing of file id " + id + " failed.", e);
+ } catch (SolrServerException e) {
+ throw new EJBException(e);
+ } catch (IOException e) {
+ throw new EJBException(e);
+ }
+ }
+
+ private void sendMetaDataOnly(CommonsHttpSolrServer solr, FileHeader file) throws SolrServerException, IOException {
+ SolrInputDocument solrDoc = new SolrInputDocument();
+ solrDoc.addField("id", file.getId().toString());
+ solrDoc.addField("name", file.getName());
+ for (FileTag t : file.getFileTags()) {
+ solrDoc.addField("tag", t.getTag());
+ }
+ for (Permission p : file.getPermissions()) {
+ if (p.getRead()) {
+ if (p.getUser() != null)
+ solrDoc.addField("ureaders", p.getUser().getId());
+ else if (p.getGroup() != null)
+ solrDoc.addField("greaders", p.getGroup().getId());
+ }
+ }
+ solrDoc.addField("owner", file.getOwner().getId());
+ solrDoc.addField("public", file.isReadForAll());
+ solr.add(solrDoc);
+ }
+
+ private String tokenizeFilename(String filename){
+ StringBuffer result = new StringBuffer();
+ StringTokenizer tokenizer = new StringTokenizer(filename,"._");
+ while(tokenizer.hasMoreTokens()){
+ result.append(tokenizer.nextToken());
+ result.append(" ");
+ }
+ result.append(filename);
+ return result.toString();
+ }
+
+ private String normalizeSearchQuery(String query) {
+ if (query.contains("*"))
+ return query.toLowerCase().replace('ά', 'α').replace('έ', 'ε').replace('ί', 'ι').replace('ή', 'η').replace('ύ', 'υ')
+ .replace('ό', 'ο').replace('ς', 'σ').replace('ώ', 'ω').replace('ϊ', 'ι').replace('ϋ', 'υ');
+ else
+ return query;
+ }
+
+ private String escapeCharacters(String text) {
+ return text.replaceAll(":", "\\\\:");
+ }
}