Node Archive and Move performance improvements

- This use case uses getPaths() instead of cycleCheck.  This is due to the nature of the archive process, but will
   be changed once cycleCheck is made to happen faster.
 - Efficient child node cache invalidation


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@31382 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/ibatis/org.hibernate.dialect.Dialect/node-common-SqlMap.xml

@@ -116,6 +116,10 @@
         <result property="typeQNameId" column="type_qname_id" jdbcType="BIGINT" javaType="java.lang.Long"/>
         <result property="assocIndex" column="assoc_index" jdbcType="INTEGER" javaType="java.lang.Integer"/>
     </resultMap>
+    <resultMap id="result_ChildAssocIds" type="ChildAssoc">
+        <result property="id" column="id" jdbcType="BIGINT" javaType="java.lang.Long"/>
+        <result property="childNode.id" column="child_node_id" jdbcType="BIGINT" javaType="java.lang.Long"/>
+    </resultMap>
     <resultMap id="result_ChildAssoc" type="ChildAssoc">
         <result property="id" column="id" jdbcType="BIGINT" javaType="java.lang.Long"/>
         <result property="parentNode.id" column="parentNodeId" jdbcType="BIGINT" javaType="java.lang.Long"/>
@@ -404,6 +408,17 @@
         </if>
     </update>
     
+    <update id="update_NodeBulkTouch" parameterType="Ids">
+        update alf_node set
+            version           = version + 1,
+            transaction_id    = #{idOne}
+        where
+            id in
+            <foreach item="item" index="index" collection="ids" open="(" separator="," close=")">
+                #{item}
+            </foreach>
+    </update>
+    
     <update id="update_NodeAssoc" parameterType="NodeAssoc">
         update alf_node_assoc set
             assoc_index = #{assocIndex}

config/alfresco/ibatis/org.hibernate.dialect.Dialect/node-select-children-SqlMap.xml

@@ -11,4 +11,20 @@
         </if>
     </select>
 
+    <select id="select_ChildNodeIds_Limited" parameterType="ChildAssoc" resultMap="result_ChildAssocIds">
+        <![CDATA[
+        select
+            ca.id               as id,
+            ca.child_node_id    as child_node_id
+        from
+            alf_child_assoc ca
+        where
+            ca.parent_node_id = #{parentNode.id} and
+            ca.is_primary = #{isPrimary} and
+            ca.id >= #{id}
+        order by
+            ca.id
+        ]]>
+    </select>
+
 </mapper>

config/alfresco/ibatis/org.hibernate.dialect.MySQLInnoDBDialect/node-select-children-SqlMap.xml

@@ -4,13 +4,32 @@
 
 <mapper namespace="alfresco.node.select.children">
 
-    <!-- Note the MySQL specific fetch size limitation (Integer.MIN_VALUE) on this statement. This activates result set streaming. -->
+    <!-- Note the MySQL specific fetch size limitation (Integer.MIN_VALUE). fetchSize activates resultset streaming. -->
+    <!-- '1=1': Makes the SQL string unique WRT the prepared statement cache -->
+
     <select id="select_ChildAssocsOfParent_Limited" parameterType="ChildAssoc" resultMap="result_ChildAssoc" fetchSize="-2147483648">
         <include refid="alfresco.node.select_ChildAssocsOfParent_Query"/>
-        and 1=1 <!-- This part present to make the SQL string unique WRT the prepared statement cache -->
+        and 1=1
         <if test="ordered == true">
             <include refid="alfresco.node.select_ChildAssoc_OrderBy"/>
         </if>
     </select>
 
+    <select id="select_ChildNodeIds_Limited" parameterType="Ids" resultMap="result_ChildAssocIds" fetchSize="-2147483648">
+        <![CDATA[
+        select
+            ca.id               as id,
+            ca.child_node_id    as child_node_id
+        from
+            alf_child_assoc ca
+        where
+            ca.parent_node_id = #{parentNode.id} and
+            ca.is_primary = #{isPrimary} and
+            ca.id >= #{id}
+            and 1=1
+        order by
+            ca.id
+        ]]>
+    </select>
+
 </mapper>

source/java/org/alfresco/repo/domain/node/AbstractNodeDAOImpl.java

@@ -85,7 +85,6 @@ import org.alfresco.util.Pair;
 import org.alfresco.util.PropertyCheck;
 import org.alfresco.util.ReadWriteLockExecuter;
 import org.alfresco.util.SerializationUtils;
-import org.apache.commons.lang.mutable.MutableInt;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.springframework.dao.ConcurrencyFailureException;
@@ -448,56 +447,55 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
      * where the child associations or nodes are modified en-masse.
      * 
      * @param parentNodeId          the parent node of all child nodes to be invalidated (may be <tt>null</tt>)
+     * @param touchNodes            <tt>true<tt> to also touch the nodes
      * @return                      the number of child associations found (might be capped)
      */
-    private int invalidateNodeChildrenCaches(Long parentNodeId)
+    private int invalidateNodeChildrenCaches(Long parentNodeId, boolean primary, boolean touchNodes)
     {
-        // Select all children
+        Long txnId = getCurrentTransaction().getId();
-        final MutableInt count = new MutableInt(0);
+
-        ChildAssocRefQueryCallback callback = new ChildAssocRefQueryCallback()
+        int count = 0;
+        List<Long> childNodeIds = new ArrayList<Long>(256);
+        Long minAssocIdInclusive = Long.MIN_VALUE;
+        while (minAssocIdInclusive != null)
         {
-            private boolean isClearOn = false;
+            childNodeIds.clear();
-            
+            List<ChildAssocEntity> childAssocs = selectChildNodeIds(
-            public boolean preLoadNodes()
+                    parentNodeId,
+                    Boolean.valueOf(primary),
+                    minAssocIdInclusive,
+                    256);
+            // Remove the cache entries as we go
+            for (ChildAssocEntity childAssoc : childAssocs)
             {
-                return false;
+                Long childAssocId = childAssoc.getId();
-            }
+                if (childAssocId.compareTo(minAssocIdInclusive) < 0)
-            
-            @Override
-            public boolean orderResults()
-            {
-                return false;
-            }
-
-            public boolean handle(
-                    Pair<Long, ChildAssociationRef> childAssocPair,
-                    Pair<Long, NodeRef> parentNodePair,
-                    Pair<Long, NodeRef> childNodePair)
-            {
-                if (isClearOn)
                 {
-                    // We have already decided to drop ALL cache entries
+                    throw new RuntimeException("Query results did not increase for assoc ID");
-                    return false;
                 }
-                else if (count.intValue() >= 1000)
+                else
                 {
-                    // That's enough.  Instead of walking thousands of entries
+                    minAssocIdInclusive = new Long(childAssocId.longValue() + 1L);
-                    // we just drop the cache at this stage
-                    AbstractNodeDAOImpl.this.clearCaches();
-                    isClearOn = true;
-                    return false;               // No more, please
                 }
-                count.increment();
+                // Invalidate the node cache
-                invalidateNodeCaches(childNodePair.getFirst());
+                Long childNodeId = childAssoc.getChildNode().getId();
-                return true;
+                childNodeIds.add(childNodeId);
+                invalidateNodeCaches(childNodeId);
+                count++;
             }
-
+            // Bring all the nodes into the transaction, if required
-            public void done()
+            if (touchNodes)
             {
-            }                               
+                updateNodes(txnId, childNodeIds);
-        };
+            }
-        selectChildAssocs(parentNodeId, null, null, null, null, null, callback);
+            // Now break out if we didn't have the full set of results
-        return count.intValue();
+            if (childAssocs.size() < 256)
+            {
+                break;
+            }
+        }
+        // Done
+        return count;
     }
 
     /**
@@ -508,10 +506,10 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
     private void invalidateNodeCaches(Long nodeId)
     {
         // Take the current value from the nodesCache and use that to invalidate the other caches
-        Pair<Long, Node> nodePair = nodesCache.getByKey(nodeId);
+        Node node = nodesCache.getValue(nodeId);
-        if (nodePair != null)
+        if (node != null)
         {
-            NodeVersionKey nodeVersionKey = nodePair.getSecond().getNodeVersionKey();
+            NodeVersionKey nodeVersionKey = node.getNodeVersionKey();
             invalidateNodeCaches(nodeVersionKey, true, true, true);
         }
         // Finally remove the node reference
@@ -1279,7 +1277,8 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
             // The new node will have new data not present in the cache, yet
             // TODO: Look to move the data in a cache-efficient way
             invalidateNodeCaches(newChildNodeId);
-            invalidateNodeChildrenCaches(newChildNodeId);
+            invalidateNodeChildrenCaches(newChildNodeId, true, true);
+            invalidateNodeChildrenCaches(newChildNodeId, false, true);
             // Now update the original to be 'deleted'
             NodeUpdateEntity childNodeUpdate = new NodeUpdateEntity();
             childNodeUpdate.setId(childNodeId);
@@ -1339,7 +1338,11 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
         childAssocRetryingHelper.doWithRetry(callback);
         
         // Check for cyclic relationships
-        cycleCheck(newChildNodeId);
+        // TODO: This adds a lot of overhead when moving hierarchies.
+        //       While getPaths is faster, it would be better to avoid the parentAssocsCache
+        //       completely.
+        getPaths(newChildNode.getNodePair(), false);
+//        cycleCheck(newChildNodeId);
 
         // Update ACLs for moved tree
         Long newParentAclId = newParentNode.getAclId();
@@ -1637,7 +1640,7 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
                 primaryParentNodeId,
                 optionalOldSharedAlcIdInAdditionToNull,
                 newSharedAclId);
-        invalidateNodeChildrenCaches(primaryParentNodeId);
+        invalidateNodeChildrenCaches(primaryParentNodeId, true, false);
     }
     
     public void deleteNode(Long nodeId)
@@ -1689,7 +1692,8 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
         deleteNodeAssocsToAndFrom(nodeId);
         
         // Remove child associations (invalidate children)
-        invalidateNodeChildrenCaches(nodeId);
+        invalidateNodeChildrenCaches(nodeId, true, true);
+        invalidateNodeChildrenCaches(nodeId, false, true);
         deleteChildAssocsToAndFrom(nodeId);
         
         // Remove aspects
@@ -2996,7 +3000,6 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
                 parentNodeId,
                 assocTypeQName,
                 assocQName,
-                null,
                 maxResults,
                 new ChildAssocRefBatchingQueryCallback(resultsCallback));
     }
@@ -3902,6 +3905,7 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
     protected abstract int updateStore(StoreEntity store);
     protected abstract Long insertNode(NodeEntity node);
     protected abstract int updateNode(NodeUpdateEntity nodeUpdate);
+    protected abstract int updateNodes(Long txnId, List<Long> nodeIds);
     protected abstract void updatePrimaryChildrenSharedAclId(
             Long txnId,
             Long primaryParentNodeId,
@@ -3947,6 +3951,11 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
     protected abstract int updateChildAssocsUniqueName(Long childNodeId, String name);
     protected abstract int deleteChildAssocsToAndFrom(Long nodeId);
     protected abstract ChildAssocEntity selectChildAssoc(Long assocId);
+    protected abstract List<ChildAssocEntity> selectChildNodeIds(
+            Long nodeId,
+            Boolean isPrimary,
+            Long minAssocIdInclusive,
+            int maxResults);
     protected abstract List<NodeIdAndAclId> selectPrimaryChildAcls(Long nodeId);
     protected abstract List<ChildAssocEntity> selectChildAssoc(
             Long parentNodeId,
@@ -3968,7 +3977,6 @@ public abstract class AbstractNodeDAOImpl implements NodeDAO, BatchingDAO
             Long parentNodeId,
             QName assocTypeQName,
             QName assocQName,
-            Long minAssocIdInclusive,
             int maxResults,
             ChildAssocRefQueryCallback resultsCallback);
     protected abstract void selectChildAssocs(

source/java/org/alfresco/repo/domain/node/ibatis/NodeDAOImpl.java

@@ -85,6 +85,7 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
     private static final String SELECT_STORE_ROOT_NODE_BY_REF = "alfresco.node.select_StoreRootNodeByRef";
     private static final String INSERT_NODE = "alfresco.node.insert.insert_Node";
     private static final String UPDATE_NODE = "alfresco.node.update_Node";
+    private static final String UPDATE_NODE_BULK_TOUCH = "alfresco.node.update_NodeBulkTouch";
     private static final String DELETE_NODE_BY_ID = "alfresco.node.delete_NodeById";
     private static final String DELETE_NODES_BY_TXN_COMMIT_TIME = "alfresco.node.delete_NodesByTxnCommitTime";
     private static final String SELECT_NODE_BY_ID = "alfresco.node.select_NodeById";
@@ -109,6 +110,7 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
     private static final String SELECT_NODE_ASSOCS_BY_TARGET = "alfresco.node.select_NodeAssocsByTarget";
     private static final String SELECT_NODE_ASSOC_BY_ID = "alfresco.node.select_NodeAssocById";
     private static final String SELECT_NODE_ASSOCS_MAX_INDEX = "alfresco.node.select_NodeAssocsMaxId";
+    private static final String SELECT_CHILD_NODE_IDS = "alfresco.node.select.children.select_ChildNodeIds_Limited";
     private static final String SELECT_NODE_PRIMARY_CHILD_ACLS = "alfresco.node.select_NodePrimaryChildAcls";
     private static final String INSERT_CHILD_ASSOC = "alfresco.node.insert.insert_ChildAssoc";
     private static final String DELETE_CHILD_ASSOC_BY_ID = "alfresco.node.delete_ChildAssocById";
@@ -317,6 +319,19 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
         return template.update(UPDATE_NODE, nodeUpdate);
     }
     
+    @Override
+    protected int updateNodes(Long txnId, List<Long> nodeIds)
+    {
+        if (nodeIds.size() == 0)
+        {
+            return 0;
+        }
+        IdsEntity ids = new IdsEntity();
+        ids.setIdOne(txnId);
+        ids.setIds(nodeIds);
+        return template.update(UPDATE_NODE_BULK_TOUCH, ids);
+    }
+
     @Override
     protected void updatePrimaryChildrenSharedAclId(
             Long txnId,
@@ -880,6 +895,25 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
         return (ChildAssocEntity) template.selectOne(SELECT_CHILD_ASSOC_BY_ID, assoc);
     }
     
+    @SuppressWarnings("unchecked")
+    @Override
+    protected List<ChildAssocEntity> selectChildNodeIds(
+            Long nodeId,
+            Boolean isPrimary,
+            Long minAssocIdInclusive,
+            int maxResults)
+    {
+        ChildAssocEntity assoc = new ChildAssocEntity();
+        NodeEntity parentNode = new NodeEntity();
+        parentNode.setId(nodeId);
+        assoc.setParentNode(parentNode);
+        assoc.setPrimary(isPrimary);
+        assoc.setId(minAssocIdInclusive);
+        
+        RowBounds rowBounds = new RowBounds(0, maxResults);
+        return (List<ChildAssocEntity>) template.selectList(SELECT_CHILD_NODE_IDS, assoc, rowBounds);
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public List<NodeIdAndAclId> selectPrimaryChildAcls(Long nodeId)
@@ -1053,7 +1087,6 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
             Long parentNodeId,
             QName assocTypeQName,
             QName assocQName,
-            Long minAssocIdInclusive,
             final int maxResults,
             ChildAssocRefQueryCallback resultsCallback)
     {
@@ -1062,7 +1095,6 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
         NodeEntity parentNode = new NodeEntity();
         parentNode.setId(parentNodeId);
         assoc.setParentNode(parentNode);
-        assoc.setId(minAssocIdInclusive);
 
         // Type QName
         if (assocTypeQName != null)