ALF-2126 - IMAP extremely slow (read: unusable) if messages are uploaded to server

- updates to the file folder service for deep listings

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@19941 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/repo/imap/AlfrescoImapFolder.java

@@ -391,6 +391,11 @@ public class AlfrescoImapFolder extends AbstractImapFolder
         {
             return "";
         }
+        
+        if (logger.isDebugEnabled())
+        {
+            logger.debug("getFullNameInternal entry");
+        }
 
         StringBuilder fullName = new StringBuilder();
         List<FileInfo> pathList;
@@ -450,6 +455,11 @@ public class AlfrescoImapFolder extends AbstractImapFolder
     @Override
     protected int getMessageCountInternal()
     {
+        if (logger.isDebugEnabled())
+        {
+            logger.debug("getMessageCountInternal entry");
+        }
+        
         if (messages.size() == 0)
         {
             List<FileInfo> fileInfos = imapService.searchMails(folderInfo.getNodeRef(), "*", viewMode, false);
@@ -552,6 +562,10 @@ public class AlfrescoImapFolder extends AbstractImapFolder
     @Override
     protected List<SimpleStoredMessage> getMessagesInternal(MsgRangeFilter msgRangeFilter)
     {
+        if (logger.isDebugEnabled())
+        {
+            logger.debug("getMessagesInternal entry");
+        }
         if (messages == null || messages.size() == 0)
         {
             List<FileInfo> fileInfos = imapService.searchMails(folderInfo.getNodeRef(), "*", viewMode, false);
@@ -565,6 +579,11 @@ public class AlfrescoImapFolder extends AbstractImapFolder
                 ret.add(messages.get(i));
             }
         }
+        
+        if (logger.isDebugEnabled())
+        {
+            logger.debug("getMessagesInternal exit");
+        }
 
         return ret;
     }

source/java/org/alfresco/repo/model/filefolder/FileFolderServiceImpl.java

@@ -28,6 +28,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.Stack;
 
 import org.alfresco.error.AlfrescoRuntimeException;
 import org.alfresco.model.ContentModel;
@@ -86,6 +87,21 @@ public class FileFolderServiceImpl implements FileFolderService
         " and (subtypeOf('" + ContentModel.TYPE_FOLDER + "') or subtypeOf('" + ContentModel.TYPE_CONTENT + "')" +
         " or subtypeOf('" + ContentModel.TYPE_LINK + "'))]";
     
+    /** Deep search for folders with a name pattern */
+    private static final String XPATH_QUERY_DEEP_FOLDERS =
+        ".//*" +
+        "[like(@cm:name, $cm:name, false)" +
+        " and not (subtypeOf('" + ContentModel.TYPE_SYSTEM_FOLDER + "'))" +
+        " and (subtypeOf('" + ContentModel.TYPE_FOLDER + "'))]";
+    
+    /** Deep search for files with a name pattern */
+    private static final String XPATH_QUERY_DEEP_FILES =
+        ".//*" +
+        "[like(@cm:name, $cm:name, false)" +
+        " and not (subtypeOf('" + ContentModel.TYPE_SYSTEM_FOLDER + "'))" +
+        " and (subtypeOf('" + ContentModel.TYPE_CONTENT + "')" +
+        " or subtypeOf('" + ContentModel.TYPE_LINK + "'))]";
+       
     /** empty parameters */
     private static final QueryParameterDefinition[] PARAMS_ANY_NAME = new QueryParameterDefinition[1];
     
@@ -422,12 +438,21 @@ public class FileFolderServiceImpl implements FileFolderService
         boolean anyName = namePattern.equals(LUCENE_MULTI_CHAR_WILDCARD);
         
         List<NodeRef> nodeRefs = null;
-        if (!includeSubFolders && anyName)
+        if (anyName)
         {
-            nodeRefs = listSimple(contextNodeRef, folderSearch, fileSearch);
+            // This is search for any name
+            if(includeSubFolders)
+            {
+               nodeRefs = listSimpleDeep(contextNodeRef, folderSearch, fileSearch);   
+            }
+            else
+            {
+                nodeRefs = listSimple(contextNodeRef, folderSearch, fileSearch);
+            }
         }
-        else                // Go with XPath
+        else                
         {
+            // TODO - we need to get rid of this xpath stuff
             // if the name pattern is null, then we use the ANY pattern
             QueryParameterDefinition[] params = null;
             if (namePattern != null)
@@ -447,16 +472,32 @@ public class FileFolderServiceImpl implements FileFolderService
             }
             else
             {
+                // name pattern is null so search for ANY_NAME
                 params = PARAMS_ANY_NAME;
             }
             // determine the correct query to use
             String query = null;
             if (includeSubFolders)
             {
-                query = XPATH_QUERY_DEEP_ALL;
+                // this is a deep search
+                if(!fileSearch && folderSearch)
+                {
+                    // This is a folder search only;
+                    query = XPATH_QUERY_DEEP_FOLDERS;
+                }
+                else if(fileSearch && folderSearch)
+                {
+                    // This is a folder search only;
+                    query = XPATH_QUERY_DEEP_FILES;
+                }
+                else
+                {        
+                    query = XPATH_QUERY_DEEP_ALL;
+                }
             }
             else
             {
+                // this is a shallow search
                 query = XPATH_QUERY_SHALLOW_ALL;
             }
             // execute the query
@@ -510,6 +551,94 @@ public class FileFolderServiceImpl implements FileFolderService
         return result;
     }
     
+    /**
+     * A deep version of listSimple.   Which recursivly walks down the tree from a given starting point, returning 
+     * the node refs found along the way.
+     * 
+     * MER: I've added this rather than changeing listSimple to minimise the risk of breaking 
+     * the existing code.   This is a quick performance improvent between using 
+     * XPath which is awful or adding new methods to the NodeService/DB   This is also a dangerous method in that it can return a 
+     * lot of data and take a long time. 
+     * 
+     * @param contextNodeRef the starting point.
+     * @param folders return nodes of type folders.
+     * @param files return nodes of type files.
+     * @return list of node refs
+     */
+    private List<NodeRef> listSimpleDeep(NodeRef contextNodeRef, boolean folders, boolean files)
+    {
+        Set<QName> folderTypeQNames = new HashSet<QName>(10);
+        Set<QName> fileTypeQNames = new HashSet<QName>(10);
+        
+        // To hold the results.
+        List<NodeRef> result = new ArrayList<NodeRef>();
+        
+        // Build a list of folder types
+        Collection<QName> qnames = dictionaryService.getSubTypes(ContentModel.TYPE_FOLDER, true);
+        folderTypeQNames.addAll(qnames);
+        folderTypeQNames.add(ContentModel.TYPE_FOLDER);
+        
+        // Remove 'system' folders
+        Collection<QName> systemFolderQNames = dictionaryService.getSubTypes(ContentModel.TYPE_SYSTEM_FOLDER, true);
+        folderTypeQNames.removeAll(systemFolderQNames);
+        folderTypeQNames.remove(ContentModel.TYPE_SYSTEM_FOLDER);
+        
+        if (files)
+        {
+            Collection<QName> fileQNames = dictionaryService.getSubTypes(ContentModel.TYPE_CONTENT, true);
+            fileTypeQNames.addAll(fileQNames);
+            fileTypeQNames.add(ContentModel.TYPE_CONTENT);
+            Collection<QName> linkQNames = dictionaryService.getSubTypes(ContentModel.TYPE_LINK, true);
+            fileTypeQNames.addAll(linkQNames);
+            fileTypeQNames.add(ContentModel.TYPE_LINK);
+        }
+        
+        if(!folders && !files)
+        {
+            return Collections.emptyList();
+            
+        }
+        
+        // Shortcut
+        if (folderTypeQNames.size() == 0)
+        {
+            return Collections.emptyList();
+        }
+        
+        Stack<NodeRef> toSearch = new Stack<NodeRef>();
+        toSearch.push(contextNodeRef);
+        
+        // Now we need to walk down the folders.
+        while(!toSearch.empty())
+        {
+            NodeRef currentDir = toSearch.pop();
+            
+            List<ChildAssociationRef> folderAssocRefs = nodeService.getChildAssocs(currentDir, folderTypeQNames);
+            
+            for (ChildAssociationRef folderRef : folderAssocRefs)
+            {
+                toSearch.push(folderRef.getChildRef());
+                if(folders)
+                {
+                    result.add(folderRef.getChildRef());
+                }
+                if(files)
+                {
+                    List<ChildAssociationRef> fileAssocRefs = nodeService.getChildAssocs(currentDir, fileTypeQNames);
+                    for (ChildAssociationRef fileRef : fileAssocRefs)
+                    {
+                        result.add(fileRef.getChildRef());
+                    }
+                }
+            }
+        }
+        
+        logger.debug("search deep finished size:" + result.size());
+ 
+        // Done
+        return result;
+    }
+    
     /**
      * @see #move(NodeRef, NodeRef, String)
      */