RM-2162: Records Management patch RMv22DODModelSeparationModulePatch taking too long with large amount of records

* added configuration property that allows existing DoD RM site to be converted to standard RM site
 * added deprecated properties back into rma namespace to avoid loss of data
 * patches to move properties into DoD namespace not executed if not required
 * removed a couple of references to moved properties that still existed
 * ensure address properties are moved if required

+review RM



git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/modules/recordsmanagement/BRANCHES/V2.2.1.x@103185 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

rm-server/config/alfresco/module/org_alfresco_module_rm/alfresco-global.properties

@@ -52,3 +52,13 @@ rm.dispositionlifecycletrigger.cronexpression=0 0/5 * * * ?
 # Indicates whether mandatory properties are checked before completing a record
 #
 rm.completerecord.mandatorypropertiescheck.enabled=true
+
+#
+# Indicates whether the existing file plan is converted to a standard file plan during
+# upgrade to V2.2, otherwise it will be converted to a DoD compliant file plan.
+# 
+# Note that when converted to a standard file plan that DoD related record meta-data remains 
+# on the individual records and will not be visible in the UI, but can be assessed via
+# deprecated model properties in the rma namespace.
+#
+rm.patch.v22.convertToStandardFilePlan=false

rm-server/config/alfresco/module/org_alfresco_module_rm/model/recordsModel.xml

@@ -735,6 +735,73 @@
                <type>d:text</type>
                <protected>true</protected>
             </property>
+            
+            <!-- DEPREACTED RECORD PROPERTIES -->
+            <!-- Note that these properties where deprecated in RM 2.2 and remain present to allow -->
+            <!-- migration to a standard compliance site without any loss of data -->
+            
+            <property name="rma:publicationDate">
+               <type>d:date</type>
+               <mandatory>false</mandatory>
+            </property>
+            <property name="rma:originator">
+               <type>d:text</type>
+               <mandatory>false</mandatory>
+               <index enabled="true">
+		          <atomic>true</atomic>
+		          <stored>false</stored>
+		          <tokenised>false</tokenised>
+		       </index>
+            </property>
+            <property name="rma:originatingOrganization">
+               <type>d:text</type>
+               <mandatory>false</mandatory>
+               <index enabled="true">
+		          <atomic>true</atomic>
+		          <stored>false</stored>
+		          <tokenised>false</tokenised>
+		       </index>
+            </property>
+            <property name="rma:mediaType">
+               <type>d:text</type>
+               <mandatory>false</mandatory>
+               <index enabled="true">
+		          <atomic>true</atomic>
+		          <stored>false</stored>
+		          <tokenised>false</tokenised>
+		       </index>
+            </property>
+            <property name="rma:format">
+               <type>d:text</type>
+               <mandatory>false</mandatory>
+               <index enabled="true">
+		          <atomic>true</atomic>
+		          <stored>false</stored>
+		          <tokenised>false</tokenised>
+		       </index>
+            </property>
+            <property name="rma:dateReceived">
+               <type>d:date</type>
+               <mandatory>false</mandatory>
+            </property>
+            <property name="rma:address">
+               <type>d:text</type>
+               <mandatory>false</mandatory>
+               <index enabled="true">
+		          <atomic>true</atomic>
+		          <stored>false</stored>
+		          <tokenised>false</tokenised>
+		       </index>
+            </property>
+            <property name="rma:otherAddress">
+               <type>d:text</type>
+               <mandatory>false</mandatory>
+               <index enabled="true">
+		          <atomic>true</atomic>
+		          <stored>false</stored>
+		          <tokenised>false</tokenised>
+		       </index>
+            </property>
 
          </properties>
 

rm-server/config/alfresco/module/org_alfresco_module_rm/patch/rm-patch-v22-context.xml

@@ -4,7 +4,6 @@
 <beans>
 
    <!-- RM v2.2 Patches -->
-
    <bean id="org_alfresco_module_rm_RMv22ReportTemplatePatch"
          parent="rm.parentModulePatch"
          class="org.alfresco.module.org_alfresco_module_rm.patch.v22.RMv22ReportTemplatePatch">
@@ -32,9 +31,9 @@
       <property name="description" value="DOD model separation module patch"/>
       <property name="fixesToSchema" value="1002"/>
       <property name="targetSchema" value="1003"/>
-      <property name="qnameDAO" ref="qnameDAO"/>
       <property name="patchDAO" ref="patchDAO"/>
       <property name="nodeDAO" ref="nodeDAO"/>
+      <property name="convertToStandardFilePlan" value="${rm.patch.v22.convertToStandardFilePlan}"/>
    </bean>
 
    <bean id="rm.dodCompliantSitePatch"
@@ -44,6 +43,7 @@
       <property name="fixesToSchema" value="1003"/>
       <property name="targetSchema" value="1004"/>
       <property name="qnameDAO" ref="qnameDAO"/>
+      <property name="convertToStandardFilePlan" value="${rm.patch.v22.convertToStandardFilePlan}"/>
    </bean>
 
    <bean id="rm.ghostOnDestroyDispositionActionPatch"
@@ -91,7 +91,7 @@
       <property name="filePlanRoleService" ref="FilePlanRoleService"/>
       <property name="authorityService" ref="AuthorityService"/>
    </bean>
-
+   
    <bean id="rm.holdCapabilityPatch"
          parent="rm.parentModulePatch"
          class="org.alfresco.module.org_alfresco_module_rm.patch.v22.RMv22HoldCapabilityPatch">
@@ -101,6 +101,6 @@
       <property name="filePlanService" ref="FilePlanService"/>
       <property name="permissionService" ref="permissionService"/>
       <property name="authorityService" ref="authorityService"/>
-   </bean>
+   </bean> 
 
 </beans>

rm-server/source/java/org/alfresco/module/org_alfresco_module_rm/dod5015/DOD5015Model.java

@@ -46,6 +46,8 @@ public interface DOD5015Model
     QName PROP_MEDIA_TYPE = QName.createQName(DOD_URI, "mediaType");
     QName PROP_FORMAT = QName.createQName(DOD_URI, "format");
     QName PROP_DATE_RECEIVED = QName.createQName(DOD_URI, "dateReceived");
+    QName PROP_ADDRESS = QName.createQName(DOD_URI, "address");
+    QName PROP_OTHER_ADDRESS = QName.createQName(DOD_URI, "otherAddress");
 
     // Scanned Record
     QName ASPECT_SCANNED_RECORD = QName.createQName(DOD_URI, "scannedRecord");

rm-server/source/java/org/alfresco/module/org_alfresco_module_rm/email/CustomEmailMappingServiceImpl.java

@@ -72,11 +72,11 @@ public class CustomEmailMappingServiceImpl extends AbstractLifecycleBean impleme
     /** Default custom mappings (TODO move to spring config) */
     private static final CustomMapping[] DEFAULT_MAPPINGS =
     {
-        new CustomMapping("Date", "rma:dateReceived"),
+        new CustomMapping("Date", "dod:dateReceived"),
-        new CustomMapping("messageTo", "rma:address"),
+        new CustomMapping("messageTo", "dod:address"),
-        new CustomMapping("messageFrom", "rma:originator"),
+        new CustomMapping("messageFrom", "dod:originator"),
-        new CustomMapping("messageSent", "rma:publicationDate"),
+        new CustomMapping("messageSent", "dod:publicationDate"),
-        new CustomMapping("messageCc", "rma:otherAddress")
+        new CustomMapping("messageCc", "dod:otherAddress")
     };
 
     /** Extractor */

rm-server/source/java/org/alfresco/module/org_alfresco_module_rm/patch/AbstractModulePatch.java

@@ -18,6 +18,8 @@
  */
 package org.alfresco.module.org_alfresco_module_rm.patch;
 
+import java.util.concurrent.TimeUnit;
+
 import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
 import org.alfresco.service.transaction.TransactionService;
 import org.apache.commons.logging.Log;
@@ -73,12 +75,12 @@ public abstract class AbstractModulePatch implements ModulePatch, BeanNameAware
         modulePatchExecuter.register(this);
     }
     
-    protected void  setTxnReadOnly(boolean txnReadOnly) 
+    public void setTxnReadOnly(boolean txnReadOnly) 
     {
         this.txnReadOnly = txnReadOnly;
     }
 
-    protected void setTxnRequiresNew(boolean txnRequiresNew) 
+    public void setTxnRequiresNew(boolean txnRequiresNew) 
     {
         this.txnRequiresNew = txnRequiresNew;
     }
@@ -229,15 +231,19 @@ public abstract class AbstractModulePatch implements ModulePatch, BeanNameAware
                                ",target=" + targetSchema);
         }
                
+        long startTime = System.nanoTime();
+        
         // do patch in transaction
         transactionService.getRetryingTransactionHelper().doInTransaction(
                 new ApplyCallback(),
                 txnReadOnly,
                 txnRequiresNew);
+
+        long elapsedTime = System.nanoTime() - startTime;
         
         if (LOGGER.isInfoEnabled())
         {
-            LOGGER.info("   ... module patch applied");
+            LOGGER.info("   ... module patch applied in " + TimeUnit.NANOSECONDS.toMillis(elapsedTime) + "ms");
         }
     }
 

rm-server/source/java/org/alfresco/module/org_alfresco_module_rm/patch/v22/RMv22DODCompliantSitePatch.java

@@ -36,6 +36,9 @@ public class RMv22DODCompliantSitePatch extends AbstractModulePatch
 {
     /** QName DAO */
     private QNameDAO qnameDAO;
+    
+    /** indicates whether we convert to a standard file plan or not */
+    private boolean convertToStandardFilePlan = false;
 
     /**
      * @param qnameDAO  QName DAO
@@ -45,24 +48,35 @@ public class RMv22DODCompliantSitePatch extends AbstractModulePatch
         this.qnameDAO = qnameDAO;
     }
     
+    /**
+     * @param convertToStandardFilePlan	convert to standard file if true, false otherwise
+     */
+    public void setConvertToStandardFilePlan(boolean convertToStandardFilePlan) 
+    {
+		this.convertToStandardFilePlan = convertToStandardFilePlan;
+	}
+    
     /**
      * @see org.alfresco.module.org_alfresco_module_rm.patch.AbstractModulePatch#applyInternal()
      */
     @Override
     public void applyInternal()
     {
-        // ensure all existing sites are of the correct type
+    	if (!convertToStandardFilePlan)
-        if (qnameDAO.getQName(RecordsManagementModel.TYPE_RM_SITE) != null &&
+    	{
-            qnameDAO.getQName(DOD5015Model.TYPE_DOD_5015_SITE) == null)
+	        // ensure all existing sites are of the correct type
-        {
+	        if (qnameDAO.getQName(RecordsManagementModel.TYPE_RM_SITE) != null &&
-            qnameDAO.updateQName(RecordsManagementModel.TYPE_RM_SITE, DOD5015Model.TYPE_DOD_5015_SITE);
+	            qnameDAO.getQName(DOD5015Model.TYPE_DOD_5015_SITE) == null)
-        }
+	        {
-        
+	            qnameDAO.updateQName(RecordsManagementModel.TYPE_RM_SITE, DOD5015Model.TYPE_DOD_5015_SITE);
-        // ensure all the existing file plans are of the correct type
+	        }
-        if (qnameDAO.getQName(RecordsManagementModel.TYPE_FILE_PLAN) != null &&
+	        
-            qnameDAO.getQName(DOD5015Model.TYPE_DOD_5015_FILE_PLAN) == null)
+	        // ensure all the existing file plans are of the correct type
-        {
+	        if (qnameDAO.getQName(RecordsManagementModel.TYPE_FILE_PLAN) != null &&
-            qnameDAO.updateQName(RecordsManagementModel.TYPE_FILE_PLAN, DOD5015Model.TYPE_DOD_5015_FILE_PLAN);
+	            qnameDAO.getQName(DOD5015Model.TYPE_DOD_5015_FILE_PLAN) == null)
-        }
+	        {
+	            qnameDAO.updateQName(RecordsManagementModel.TYPE_FILE_PLAN, DOD5015Model.TYPE_DOD_5015_FILE_PLAN);
+	        }
+    	}
     }
 }

rm-server/source/java/org/alfresco/module/org_alfresco_module_rm/patch/v22/RMv22DODModelSeparationModulePatch.java

@@ -20,7 +20,6 @@ package org.alfresco.module.org_alfresco_module_rm.patch.v22;
 
 import java.io.Serializable;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 import org.alfresco.module.org_alfresco_module_rm.dod5015.DOD5015Model;
@@ -28,7 +27,8 @@ import org.alfresco.module.org_alfresco_module_rm.model.RecordsManagementModel;
 import org.alfresco.module.org_alfresco_module_rm.patch.AbstractModulePatch;
 import org.alfresco.repo.domain.node.NodeDAO;
 import org.alfresco.repo.domain.patch.PatchDAO;
-import org.alfresco.repo.domain.qname.QNameDAO;
+import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
+import org.alfresco.service.cmr.repository.NodeRef;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.util.Pair;
 
@@ -43,9 +43,9 @@ public class RMv22DODModelSeparationModulePatch extends AbstractModulePatch
 {
     /** query batch size */
     private static final long BATCH_SIZE = 100000L;
-
+    
-    /** QName DAO */
+    /** indicates whether we convert to a standard file plan or not */
-    private QNameDAO qnameDAO;
+    private boolean convertToStandardFilePlan = false;
 
     /** Patch DAO */
     private PatchDAO patchDAO;
@@ -61,16 +61,18 @@ public class RMv22DODModelSeparationModulePatch extends AbstractModulePatch
         DOD5015Model.PROP_PUBLICATION_DATE,
         DOD5015Model.PROP_MEDIA_TYPE,
         DOD5015Model.PROP_FORMAT,
-        DOD5015Model.PROP_DATE_RECEIVED
+        DOD5015Model.PROP_DATE_RECEIVED,
+        DOD5015Model.PROP_ADDRESS,
+        DOD5015Model.PROP_OTHER_ADDRESS
     };
-
+    
     /**
-     * @param qnameDAO  QName DAO
+     * @param convertToStandardFilePlan	convert to standard file if true, false otherwise
      */
-    public void setQnameDAO(QNameDAO qnameDAO)
+    public void setConvertToStandardFilePlan(boolean convertToStandardFilePlan) 
     {
-        this.qnameDAO = qnameDAO;
+		this.convertToStandardFilePlan = convertToStandardFilePlan;
-    }
+	}
 
     /**
      * @param patchDAO  patch DAO
@@ -94,50 +96,73 @@ public class RMv22DODModelSeparationModulePatch extends AbstractModulePatch
     @Override
     public void applyInternal()
     {
-        Long maxNodeId = patchDAO.getMaxAdmNodeID();
+    	if (!convertToStandardFilePlan)
-        long recordCount = patchDAO.getCountNodesWithAspects(Collections.singleton(ASPECT_RECORD));
+    	{
-        if (LOGGER.isDebugEnabled())
+	        Long maxNodeId = patchDAO.getMaxAdmNodeID();
-        {
+	        long recordCount = patchDAO.getCountNodesWithAspects(Collections.singleton(ASPECT_RECORD));
-            LOGGER.debug(" ... updating " + recordCount + " records");
+	        if (LOGGER.isDebugEnabled())
-        }
+	        {
+	            LOGGER.debug("   ... updating " + recordCount + " records in batches of " + BATCH_SIZE);
+	        }
+	
+	        // apply the DOD record aspect to all exiting records
+	        int completed = 0;
+	        for (Long i = 0L; i < maxNodeId; i+=BATCH_SIZE)
+	        {
+	        	final Long finali = i;
+	        	Integer batchCount = transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback<Integer>()
+	            {
+	        		int batchCount = 0;
+	        		
+	        		public Integer execute() throws Throwable 
+					{
+						nodeDAO.getNodesWithAspects(Collections.singleton(ASPECT_RECORD), finali, finali + BATCH_SIZE, new NodeDAO.NodeRefQueryCallback()
+				        {	
+				            public boolean handle(Pair<Long, NodeRef> nodePair)
+				            {
+				            	 // get the records properties
+				            	 Map<QName, Serializable> properties = nodeDAO.getNodeProperties(nodePair.getFirst());
+				            	 boolean changed = false;
 
-        // apply the DOD record aspect to all exiting records
+				            	 for (QName qname : qnames)
-        int completed = 0;
+				            	 {
-        Pair<Long, QName> recordAspect = qnameDAO.getQName(ASPECT_RECORD);
+				            	     // if the record has any of the moved properties
-        if (recordAspect != null)
+				            	     QName origional = QName.createQName(RecordsManagementModel.RM_URI, qname.getLocalName());
-        {
+				            	     if (properties.containsKey(origional))
-            for (Long i = 0L; i < maxNodeId; i+=BATCH_SIZE)
+				            	     {
-            {
+				            	         // move the property value
-                List<Long> nodeIds = patchDAO.getNodesByAspectQNameId(recordAspect.getFirst(), i, i + BATCH_SIZE);
+				            	         Serializable value = properties.get(origional);
-                for (Long nodeId : nodeIds)
+				            	         properties.put(qname, value);
-                {
+				            	         properties.remove(origional);
-                    // get the records properties
+				            	         changed = true;
-                    Map<QName, Serializable> properties = nodeDAO.getNodeProperties(nodeId);
+				            	     }
-                    
+				            	 }
-                    for (QName qname : qnames)
-                    {
-                        // if the record has any of the moved properties
-                        QName origional = QName.createQName(RecordsManagementModel.RM_URI, qname.getLocalName());
-                        if (properties.containsKey(origional))
-                        {
-                            // move the property value
-                            Serializable value = properties.get(origional);
-                            properties.put(qname, value);
-                            properties.remove(origional);
-                        }
-                    }
-                    
-                    // set properties and add aspect
-                    nodeDAO.setNodeProperties(nodeId, properties);
-                    nodeDAO.addNodeAspects(nodeId, Collections.singleton(DOD5015Model.ASPECT_DOD_5015_RECORD));
-                }
 
-                completed += completed + nodeIds.size();
+				            	 // set properties and add aspect
-                if (LOGGER.isDebugEnabled())
+				            	 if (changed)
-                {
+				            	 {
-                    LOGGER.debug("    ... completed " + completed + " of " + recordCount);
+				            		 nodeDAO.setNodeProperties(nodePair.getFirst(), properties);
-                }
+				            	 }				            	
-            }
+				            	 nodeDAO.addNodeAspects(nodePair.getFirst(), Collections.singleton(DOD5015Model.ASPECT_DOD_5015_RECORD));
-        }
+				            	 batchCount ++;
+				                
+				            	 return true;
+				            }
+				        });
+				
+						return batchCount;
+					}           		
+	            } , false, true);    
+	        	
+	        	if (batchCount != 0)
+	        	{
+		        	completed = completed + batchCount;
+		            if (LOGGER.isDebugEnabled())
+		            {
+		                LOGGER.debug("   ... completed " + completed + " of " + recordCount);
+		            }
+	        	}
+	        }
+    	}
     }
 }