ALF-11957: Merged PATCHES/V3.4.6 to HEAD

32617: ALF-11879: IMAP performance
      - Fix node batch loading - batch load ContentData to avoid N+1 problem with content properties
      - During cache preloading, use distinct transactions for each folder search, thus avoiding blowing the transactional caches
   32619: ALF-11879: Fixed typo
   32652: ALF-11879: Deactivate auto-versioning and auditing (and run as system) whilst setting magic IMAP aspect properties


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@32673 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/ibatis/org.hibernate.dialect.Dialect/content-common-SqlMap.xml

@@ -264,6 +264,26 @@
             cd.id = ?
     </select>
 
+    <!-- Get ContentData entities by node ID -->
+    <select id="select_ContentDataByNodeIds" parameterType="Ids" resultMap="result_ContentData">
+        select
+            cd.id as id,
+            cd.version as version,
+            cd.content_url_id as content_url_id,
+            cu.content_url as content_url,
+            cu.content_size as content_size,
+            cd.content_mimetype_id as content_mimetype_id,
+            cd.content_encoding_id as content_encoding_id,
+            cd.content_locale_id as content_locale_id
+        from
+            alf_content_data cd
+            left join alf_content_url cu on (cd.content_url_id = cu.id)
+            left join alf_node_properties np on (cd.id = np.long_value)
+        where
+            np.node_id in <iterate property="ids" open="(" close=")" conjunction=",">#ids[]#</iterate> and
+            (np.actual_type_n = 3 or np.actual_type_n = 21)
+    </select>
+
     <!-- Get the ContentData entity by Node and property QName -->
     <select id="select_ContentDataByNodeAndQName" parameterType="Ids" resultType="long">
         select

source/java/org/alfresco/repo/domain/contentdata/AbstractContentDataDAOImpl.java

@@ -179,6 +179,14 @@ public abstract class AbstractContentDataDAOImpl implements ContentDataDAO
         return entityPair;
     }
 
+    public void cacheContentDataForNodes(Set<Long> nodeIds)
+    {
+        for (ContentDataEntity entity : getContentDataEntitiesForNodes(nodeIds))
+        {
+            contentDataCache.setValue(entity.getId(), makeContentData(entity));
+        }        
+    }
+
     /**
      * {@inheritDoc}
      */
@@ -494,6 +502,13 @@ public abstract class AbstractContentDataDAOImpl implements ContentDataDAO
      */
     protected abstract ContentDataEntity getContentDataEntity(Long id);
     
+    
+    /**
+     * @param nodeIds       the node ID
+     * @return              Returns the associated entities or <tt>null</tt> if none exist
+     */
+    protected abstract List<ContentDataEntity> getContentDataEntitiesForNodes(Set<Long> nodeIds);    
+
     /**
      * Update an existing <b>alf_content_data</b> entity
      * 

source/java/org/alfresco/repo/domain/contentdata/ContentDataDAO.java

@@ -69,6 +69,12 @@ public interface ContentDataDAO
      */
     Pair<Long, ContentData> getContentData(Long id);
     
+    /**
+     * @param nodeIds       the nodeIds
+     * @throws              AlfrescoRuntimeException if an ID provided is invalid
+     */
+    public void cacheContentDataForNodes(Set<Long> nodeIds);
+
     /**
      * Delete an instance of content.
      * @param id            the unique ID of the entity

source/java/org/alfresco/repo/domain/contentdata/ibatis/ContentDataDAOImpl.java

@@ -19,6 +19,7 @@
 package org.alfresco.repo.domain.contentdata.ibatis;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -54,6 +55,7 @@ public class ContentDataDAOImpl extends AbstractContentDataDAOImpl
     private static final String SELECT_CONTENT_URLS_ORPHANED = "alfresco.content.select_ContentUrlsOrphaned";
     private static final String SELECT_CONTENT_DATA_BY_ID = "alfresco.content.select_ContentDataById";
     private static final String SELECT_CONTENT_DATA_BY_NODE_AND_QNAME = "alfresco.content.select_ContentDataByNodeAndQName";
+    private static final String SELECT_CONTENT_DATA_BY_NODE_IDS = "alfresco.content.select_ContentDataByNodeIds";
     private static final String INSERT_CONTENT_URL = "alfresco.content.insert.insert_ContentUrl";
     private static final String INSERT_CONTENT_DATA = "alfresco.content.insert.insert_ContentData";
     private static final String UPDATE_CONTENT_URL_ORPHAN_TIME = "alfresco.content.update_ContentUrlOrphanTime";
@@ -209,6 +211,20 @@ public class ContentDataDAOImpl extends AbstractContentDataDAOImpl
         return contentDataEntity;
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    protected List<ContentDataEntity> getContentDataEntitiesForNodes(Set<Long> nodeIds)
+    {
+        if (nodeIds.size() == 0)
+        {
+            // There will be no results
+            return Collections.emptyList();
+        }
+        IdsEntity idsEntity = new IdsEntity();
+        idsEntity.setIds(new ArrayList<Long>(nodeIds));
+        return template.queryForList(SELECT_CONTENT_DATA_BY_NODE_IDS, idsEntity);
+    }
+    
     @Override
     protected int updateContentDataEntity(ContentDataEntity entity)
     {

source/java/org/alfresco/repo/domain/node/AbstractNodeDAOImpl.java

@@ -3902,6 +3902,10 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
             setNodeAspectsCached(nodeId, Collections.<QName>emptySet());
         }
 
+        // First ensure all content data are pre-cached, so we don't have to load them individually when converting properties
+        contentDataDAO.cacheContentDataForNodes(propertiesNodeIds);
+        
+        // Now bulk load the properties
         Map<NodeVersionKey, Map<NodePropertyKey, NodePropertyValue>> propsByNodeId = selectNodeProperties(propertiesNodeIds);
         for (Map.Entry<NodeVersionKey, Map<NodePropertyKey, NodePropertyValue>> entry : propsByNodeId.entrySet())
         {

source/java/org/alfresco/repo/imap/ImapServiceImpl.java

@@ -411,17 +411,7 @@ public class ImapServiceImpl implements ImapService, OnCreateChildAssociationPol
         }
         
         // Locate or create IMAP home
-        imapHomeNodeRef = imapHomeConfigBean.getOrCreateFolderPath(namespaceService, nodeService, searchService, fileFolderService);
+        imapHomeNodeRef = imapHomeConfigBean.getOrCreateFolderPath(namespaceService, nodeService, searchService, fileFolderService);        
-        
-        // Hit the mount points and warm the caches for early failure
-        for (String mountPointName : imapConfigMountPoints.keySet())
-        {
-            for (AlfrescoImapFolder mailbox : listMailboxes(new AlfrescoImapUser(null, AuthenticationUtil
-                    .getSystemUserName(), null), mountPointName + "*", false))
-            {
-                mailbox.getUidNext();
-            }
-        }
     }
 
     public void shutdown()
@@ -437,16 +427,34 @@ public class ImapServiceImpl implements ImapService, OnCreateChildAssociationPol
                 @Override
                 public Void doWork() throws Exception
                 {
-                    return serviceRegistry.getTransactionService().getRetryingTransactionHelper().doInTransaction(
+                    List<AlfrescoImapFolder> mailboxes = serviceRegistry.getTransactionService().getRetryingTransactionHelper().doInTransaction(
-                            new RetryingTransactionCallback<Void>()
+                            new RetryingTransactionCallback<List<AlfrescoImapFolder>>()
                             {
                                 @Override
-                                public Void execute() throws Throwable
+                                public List<AlfrescoImapFolder> execute() throws Throwable
                                 {
                                     startup();
-                                    return null;
+                                    
+                                    List<AlfrescoImapFolder> result = new LinkedList<AlfrescoImapFolder>();
+                                    
+                                    // Hit the mount points and warm the caches for early failure
+                                    for (String mountPointName : imapConfigMountPoints.keySet())
+                                    {
+                                        result.addAll(listMailboxes(new AlfrescoImapUser(null, AuthenticationUtil
+                                                .getSystemUserName(), null), mountPointName + "*", false));
+                                    }
+                                    
+                                    return result;
                                 }
                             });
+                    
+                    // Let each mailbox search trigger its own distinct transaction
+                    for (AlfrescoImapFolder mailbox : mailboxes)
+                    {
+                        mailbox.getUidNext();
+                    }
+                    
+                    return null;                    
                 }
             }, AuthenticationUtil.getSystemUserName());
         }
@@ -802,7 +810,7 @@ public class ImapServiceImpl implements ImapService, OnCreateChildAssociationPol
             changeToken = GUID.generate();
             cacheKey = new Pair<String, String>(userName, changeToken);
             final String finalToken = changeToken;
-            AuthenticationUtil.runAs(new RunAsWork<Void>()
+            doAsSystem(new RunAsWork<Void>()
             {
                 @Override
                 public Void doWork() throws Exception
@@ -812,7 +820,7 @@ public class ImapServiceImpl implements ImapService, OnCreateChildAssociationPol
                             : currentSearch.lastKey());
                     return null;
                 }
-            }, AuthenticationUtil.getSystemUserName());
+            });
         }
         Long uidValidity = (Long) nodeService.getProperty(contextNodeRef, ImapModel.PROP_UIDVALIDITY);
         FolderStatus result = new FolderStatus(messageCount, recentCount, firstUnseen, unseenCount,
@@ -1524,65 +1532,67 @@ public class ImapServiceImpl implements ImapService, OnCreateChildAssociationPol
     }
 
     @Override
-    public void onCreateChildAssociation(ChildAssociationRef childAssocRef, boolean isNewNode)
+    public void onCreateChildAssociation(final ChildAssociationRef childAssocRef, boolean isNewNode)
     {
-        NodeRef childNodeRef = childAssocRef.getChildRef();
+        doAsSystem(new RunAsWork<Void>()
-        
-        if (this.serviceRegistry.getDictionaryService().isSubClass(this.nodeService.getType(childNodeRef), ContentModel.TYPE_CONTENT))
         {
-            long newId = (Long) nodeService.getProperty(childNodeRef, ContentModel.PROP_NODE_DBID);
+            @Override
-            // Keep a record of minimum and maximum node IDs in this folder in this transaction and add a listener that will
+            public Void doWork() throws Exception
-            // update the UIDVALIDITY and MAXUID properties appropriately. Also force generation of a new change token
-            getUidValidityTransactionListener(childAssocRef.getParentRef()).recordNewUid(newId);
-            // Flag new content as recent
-            setFlag(childNodeRef, Flags.Flag.RECENT, true);
-        }
-        
-        if (logger.isDebugEnabled())
-        {
-            logger.debug("[onCreateChildAssociation] Association " + childAssocRef + " created. CHANGETOKEN will be changed.");
-        }
-    }
-    
-    @Override
-    public void onDeleteChildAssociation(ChildAssociationRef childAssocRef)
-    {
-        NodeRef childNodeRef = childAssocRef.getChildRef();
-        if (this.serviceRegistry.getDictionaryService().isSubClass(this.nodeService.getType(childNodeRef), ContentModel.TYPE_CONTENT))
-        {
-            // Force generation of a new change token
-            getUidValidityTransactionListener(childAssocRef.getParentRef());
-
-            // Remove the message from the cache
-            this.messageCache.remove(childNodeRef);
-        }
-        if (logger.isDebugEnabled())
-        {
-            logger.debug("[onDeleteChildAssociation] Association " + childAssocRef + " created. CHANGETOKEN will be changed.");
-        }
-    }
-    
-    @Override
-    public void onUpdateProperties(NodeRef nodeRef, Map<QName, Serializable> before, Map<QName, Serializable> after)
-    {
-        for (ChildAssociationRef parentAssoc : nodeService.getParentAssocs(nodeRef))
-        {
-            NodeRef folderRef = parentAssoc.getParentRef();
-            if (this.nodeService.hasAspect(folderRef, ImapModel.ASPECT_IMAP_FOLDER))
             {
-                this.messageCache.remove(nodeRef);
+                NodeRef childNodeRef = childAssocRef.getChildRef();
-
+                
-                // Force generation of a new change token
+                if (serviceRegistry.getDictionaryService().isSubClass(nodeService.getType(childNodeRef), ContentModel.TYPE_CONTENT))
-                getUidValidityTransactionListener(folderRef);                
+                {
+                    long newId = (Long) nodeService.getProperty(childNodeRef, ContentModel.PROP_NODE_DBID);
+                    // Keep a record of minimum and maximum node IDs in this folder in this transaction and add a listener that will
+                    // update the UIDVALIDITY and MAXUID properties appropriately. Also force generation of a new change token
+                    getUidValidityTransactionListener(childAssocRef.getParentRef()).recordNewUid(newId);
+                    // Flag new content as recent
+                    setFlag(childNodeRef, Flags.Flag.RECENT, true);
+                }
+                
+                if (logger.isDebugEnabled())
+                {
+                    logger.debug("[onCreateChildAssociation] Association " + childAssocRef + " created. CHANGETOKEN will be changed.");
+                }
+                return null;
             }
-        }
+        });
     }
-
+    
     @Override
-    public void beforeDeleteNode(final NodeRef nodeRef)
+    public void onDeleteChildAssociation(final ChildAssociationRef childAssocRef)
     {
-        // RUN AS SYSTEM due to Node Service archive permissions problem ALF-11103
+        doAsSystem(new RunAsWork<Void>()
-        AuthenticationUtil.runAs(new RunAsWork<Void>()
+        {
+            @Override
+            public Void doWork() throws Exception
+            {
+                NodeRef childNodeRef = childAssocRef.getChildRef();
+                if (serviceRegistry.getDictionaryService().isSubClass(nodeService.getType(childNodeRef),
+                        ContentModel.TYPE_CONTENT))
+                {
+                    // Force generation of a new change token
+                    getUidValidityTransactionListener(childAssocRef.getParentRef());
+
+                    // Remove the message from the cache
+                    messageCache.remove(childNodeRef);
+                }
+                if (logger.isDebugEnabled())
+                {
+                    logger.debug("[onDeleteChildAssociation] Association " + childAssocRef
+                            + " created. CHANGETOKEN will be changed.");
+                }
+                return null;
+            }
+        });
+    }
+    
+    @Override
+    public void onUpdateProperties(final NodeRef nodeRef, Map<QName, Serializable> before,
+            Map<QName, Serializable> after)
+    {
+        doAsSystem(new RunAsWork<Void>()
         {
             @Override
             public Void doWork() throws Exception
@@ -1595,9 +1605,52 @@ public class ImapServiceImpl implements ImapService, OnCreateChildAssociationPol
                         messageCache.remove(nodeRef);
 
                         // Force generation of a new change token
-                        getUidValidityTransactionListener(folderRef);                
+                        getUidValidityTransactionListener(folderRef);
                     }
                 }
+                return null;
+            }
+        });
+    }
+
+    @Override
+    public void beforeDeleteNode(final NodeRef nodeRef)
+    {
+        // RUN AS SYSTEM due to Node Service archive permissions problem ALF-11103
+        doAsSystem(new RunAsWork<Void>()
+        {
+            @Override
+            public Void doWork() throws Exception
+            {
+                for (ChildAssociationRef parentAssoc : nodeService.getParentAssocs(nodeRef))
+                {
+                    NodeRef folderRef = parentAssoc.getParentRef();
+                    if (nodeService.hasAspect(folderRef, ImapModel.ASPECT_IMAP_FOLDER))
+                    {
+                        messageCache.remove(nodeRef);
+
+                        // Force generation of a new change token
+                getUidValidityTransactionListener(folderRef);                
+                    }
+                }
+                return null;
+                    }
+        });
+    }
+
+    private <R> R doAsSystem(RunAsWork<R> work)
+    {
+        policyBehaviourFilter.disableBehaviour(ContentModel.ASPECT_AUDITABLE);
+        policyBehaviourFilter.disableBehaviour(ContentModel.ASPECT_VERSIONABLE);
+        try
+        {
+            return AuthenticationUtil.runAs(work, AuthenticationUtil.getSystemUserName());
+        }
+        finally
+        {
+            policyBehaviourFilter.enableBehaviour(ContentModel.ASPECT_AUDITABLE);
+            policyBehaviourFilter.enableBehaviour(ContentModel.ASPECT_VERSIONABLE);
+                }
         
                 return null;
             }
@@ -1643,7 +1696,7 @@ public class ImapServiceImpl implements ImapService, OnCreateChildAssociationPol
                 return;
             }
 
-            AuthenticationUtil.runAs(new RunAsWork<Void>()
+            doAsSystem(new RunAsWork<Void>()
             {
                 @Override
                 public Void doWork() throws Exception
@@ -1670,7 +1723,7 @@ public class ImapServiceImpl implements ImapService, OnCreateChildAssociationPol
                     UidValidityTransactionListener.this.nodeService.setProperty(folderNodeRef, ImapModel.PROP_CHANGE_TOKEN, changeToken);                            
                     return null;
                 }                        
-            }, AuthenticationUtil.getSystemUserName());
+            });
         }
     }