inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-07-31 17:39:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

RM-2162: Records Management patch RMv22DODModelSeparationModulePatch taking too long with large amount of records

Browse Source 
* added configuration property that allows existing DoD RM site to be converted to standard RM site
 * added deprecated properties back into rma namespace to avoid loss of data
 * patches to move properties into DoD namespace not executed if not required
 * removed a couple of references to moved properties that still existed
 * ensure address properties are moved if required

+review RM



git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/modules/recordsmanagement/BRANCHES/V2.2.1.x@103185 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Roy Wetherall
2015-04-30 04:42:25 +00:00
parent b478cc7a6a
commit 2541cc36ab
8 changed files with 203 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
rm-server/config/alfresco/module/org_alfresco_module_rm/alfresco-global.properties
View File

@@ -52,3 +52,13 @@ rm.dispositionlifecycletrigger.cronexpression=0 0/5 * * * ?
# Indicates whether mandatory properties are checked before completing a record
#
rm.completerecord.mandatorypropertiescheck.enabled=true
#
# Indicates whether the existing file plan is converted to a standard file plan during
# upgrade to V2.2, otherwise it will be converted to a DoD compliant file plan.
#
# Note that when converted to a standard file plan that DoD related record meta-data remains
# on the individual records and will not be visible in the UI, but can be assessed via
# deprecated model properties in the rma namespace.
#
rm.patch.v22.convertToStandardFilePlan=false

67
rm-server/config/alfresco/module/org_alfresco_module_rm/model/recordsModel.xml
View File

@@ -736,6 +736,73 @@
<protected>true</protected>
</property>
<!-- DEPREACTED RECORD PROPERTIES -->
<!-- Note that these properties where deprecated in RM 2.2 and remain present to allow -->
<!-- migration to a standard compliance site without any loss of data -->
<property name="rma:publicationDate">
<type>d:date</type>
<mandatory>false</mandatory>
</property>
<property name="rma:originator">
<type>d:text</type>
<mandatory>false</mandatory>
<index enabled="true">
<atomic>true</atomic>
<stored>false</stored>
<tokenised>false</tokenised>
</index>
</property>
<property name="rma:originatingOrganization">
<type>d:text</type>
<mandatory>false</mandatory>
<index enabled="true">
<atomic>true</atomic>
<stored>false</stored>
<tokenised>false</tokenised>
</index>
</property>
<property name="rma:mediaType">
<type>d:text</type>
<mandatory>false</mandatory>
<index enabled="true">
<atomic>true</atomic>
<stored>false</stored>
<tokenised>false</tokenised>
</index>
</property>
<property name="rma:format">
<type>d:text</type>
<mandatory>false</mandatory>
<index enabled="true">
<atomic>true</atomic>
<stored>false</stored>
<tokenised>false</tokenised>
</index>
</property>
<property name="rma:dateReceived">
<type>d:date</type>
<mandatory>false</mandatory>
</property>
<property name="rma:address">
<type>d:text</type>
<mandatory>false</mandatory>
<index enabled="true">
<atomic>true</atomic>
<stored>false</stored>
<tokenised>false</tokenised>
</index>
</property>
<property name="rma:otherAddress">
<type>d:text</type>
<mandatory>false</mandatory>
<index enabled="true">
<atomic>true</atomic>
<stored>false</stored>
<tokenised>false</tokenised>
</index>
</property>
</properties>
<mandatory-aspects>

4
rm-server/config/alfresco/module/org_alfresco_module_rm/patch/rm-patch-v22-context.xml
View File

@@ -4,7 +4,6 @@
<beans>
<!-- RM v2.2 Patches -->
<bean id="org_alfresco_module_rm_RMv22ReportTemplatePatch"
parent="rm.parentModulePatch"
class="org.alfresco.module.org_alfresco_module_rm.patch.v22.RMv22ReportTemplatePatch">
@@ -32,9 +31,9 @@
<property name="description" value="DOD model separation module patch"/>
<property name="fixesToSchema" value="1002"/>
<property name="targetSchema" value="1003"/>
<property name="qnameDAO" ref="qnameDAO"/>
<property name="patchDAO" ref="patchDAO"/>
<property name="nodeDAO" ref="nodeDAO"/>
<property name="convertToStandardFilePlan" value="${rm.patch.v22.convertToStandardFilePlan}"/>
</bean>
<bean id="rm.dodCompliantSitePatch"
@@ -44,6 +43,7 @@
<property name="fixesToSchema" value="1003"/>
<property name="targetSchema" value="1004"/>
<property name="qnameDAO" ref="qnameDAO"/>
<property name="convertToStandardFilePlan" value="${rm.patch.v22.convertToStandardFilePlan}"/>
</bean>
<bean id="rm.ghostOnDestroyDispositionActionPatch"

2
rm-server/source/java/org/alfresco/module/org_alfresco_module_rm/dod5015/DOD5015Model.java
View File

@@ -46,6 +46,8 @@ public interface DOD5015Model
QName PROP_MEDIA_TYPE = QName.createQName(DOD_URI, "mediaType");
QName PROP_FORMAT = QName.createQName(DOD_URI, "format");
QName PROP_DATE_RECEIVED = QName.createQName(DOD_URI, "dateReceived");
QName PROP_ADDRESS = QName.createQName(DOD_URI, "address");
QName PROP_OTHER_ADDRESS = QName.createQName(DOD_URI, "otherAddress");
// Scanned Record
QName ASPECT_SCANNED_RECORD = QName.createQName(DOD_URI, "scannedRecord");

10
rm-server/source/java/org/alfresco/module/org_alfresco_module_rm/email/CustomEmailMappingServiceImpl.java
View File

@@ -72,11 +72,11 @@ public class CustomEmailMappingServiceImpl extends AbstractLifecycleBean impleme
/** Default custom mappings (TODO move to spring config) */
private static final CustomMapping[] DEFAULT_MAPPINGS =
{
new CustomMapping("Date", "rma:dateReceived"),
new CustomMapping("messageTo", "rma:address"),
new CustomMapping("messageFrom", "rma:originator"),
new CustomMapping("messageSent", "rma:publicationDate"),
new CustomMapping("messageCc", "rma:otherAddress")
new CustomMapping("Date", "dod:dateReceived"),
new CustomMapping("messageTo", "dod:address"),
new CustomMapping("messageFrom", "dod:originator"),
new CustomMapping("messageSent", "dod:publicationDate"),
new CustomMapping("messageCc", "dod:otherAddress")
};
/** Extractor */

12
rm-server/source/java/org/alfresco/module/org_alfresco_module_rm/patch/AbstractModulePatch.java
View File

@@ -18,6 +18,8 @@
*/
package org.alfresco.module.org_alfresco_module_rm.patch;
import java.util.concurrent.TimeUnit;
import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
import org.alfresco.service.transaction.TransactionService;
import org.apache.commons.logging.Log;
@@ -73,12 +75,12 @@ public abstract class AbstractModulePatch implements ModulePatch, BeanNameAware
modulePatchExecuter.register(this);
}
protected void setTxnReadOnly(boolean txnReadOnly)
public void setTxnReadOnly(boolean txnReadOnly)
{
this.txnReadOnly = txnReadOnly;
}
protected void setTxnRequiresNew(boolean txnRequiresNew)
public void setTxnRequiresNew(boolean txnRequiresNew)
{
this.txnRequiresNew = txnRequiresNew;
}
@@ -229,15 +231,19 @@ public abstract class AbstractModulePatch implements ModulePatch, BeanNameAware
",target=" + targetSchema);
}
long startTime = System.nanoTime();
// do patch in transaction
transactionService.getRetryingTransactionHelper().doInTransaction(
new ApplyCallback(),
txnReadOnly,
txnRequiresNew);
long elapsedTime = System.nanoTime() - startTime;
if (LOGGER.isInfoEnabled())
{
LOGGER.info(" ... module patch applied");
LOGGER.info(" ... module patch applied in " + TimeUnit.NANOSECONDS.toMillis(elapsedTime) + "ms");
}
}

38
rm-server/source/java/org/alfresco/module/org_alfresco_module_rm/patch/v22/RMv22DODCompliantSitePatch.java
View File

@@ -37,6 +37,9 @@ public class RMv22DODCompliantSitePatch extends AbstractModulePatch
/** QName DAO */
private QNameDAO qnameDAO;
/** indicates whether we convert to a standard file plan or not */
private boolean convertToStandardFilePlan = false;
/**
* @param qnameDAO QName DAO
*/
@@ -45,24 +48,35 @@ public class RMv22DODCompliantSitePatch extends AbstractModulePatch
this.qnameDAO = qnameDAO;
}
/**
* @param convertToStandardFilePlan convert to standard file if true, false otherwise
*/
public void setConvertToStandardFilePlan(boolean convertToStandardFilePlan)
{
this.convertToStandardFilePlan = convertToStandardFilePlan;
}
/**
* @see org.alfresco.module.org_alfresco_module_rm.patch.AbstractModulePatch#applyInternal()
*/
@Override
public void applyInternal()
{
// ensure all existing sites are of the correct type
if (qnameDAO.getQName(RecordsManagementModel.TYPE_RM_SITE) != null &&
qnameDAO.getQName(DOD5015Model.TYPE_DOD_5015_SITE) == null)
{
qnameDAO.updateQName(RecordsManagementModel.TYPE_RM_SITE, DOD5015Model.TYPE_DOD_5015_SITE);
}
if (!convertToStandardFilePlan)
{
// ensure all existing sites are of the correct type
if (qnameDAO.getQName(RecordsManagementModel.TYPE_RM_SITE) != null &&
qnameDAO.getQName(DOD5015Model.TYPE_DOD_5015_SITE) == null)
{
qnameDAO.updateQName(RecordsManagementModel.TYPE_RM_SITE, DOD5015Model.TYPE_DOD_5015_SITE);
}
// ensure all the existing file plans are of the correct type
if (qnameDAO.getQName(RecordsManagementModel.TYPE_FILE_PLAN) != null &&
qnameDAO.getQName(DOD5015Model.TYPE_DOD_5015_FILE_PLAN) == null)
{
qnameDAO.updateQName(RecordsManagementModel.TYPE_FILE_PLAN, DOD5015Model.TYPE_DOD_5015_FILE_PLAN);
}
// ensure all the existing file plans are of the correct type
if (qnameDAO.getQName(RecordsManagementModel.TYPE_FILE_PLAN) != null &&
qnameDAO.getQName(DOD5015Model.TYPE_DOD_5015_FILE_PLAN) == null)
{
qnameDAO.updateQName(RecordsManagementModel.TYPE_FILE_PLAN, DOD5015Model.TYPE_DOD_5015_FILE_PLAN);
}
}
}
}

125
rm-server/source/java/org/alfresco/module/org_alfresco_module_rm/patch/v22/RMv22DODModelSeparationModulePatch.java
View File

@@ -20,7 +20,6 @@ package org.alfresco.module.org_alfresco_module_rm.patch.v22;
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.alfresco.module.org_alfresco_module_rm.dod5015.DOD5015Model;
@@ -28,7 +27,8 @@ import org.alfresco.module.org_alfresco_module_rm.model.RecordsManagementModel;
import org.alfresco.module.org_alfresco_module_rm.patch.AbstractModulePatch;
import org.alfresco.repo.domain.node.NodeDAO;
import org.alfresco.repo.domain.patch.PatchDAO;
import org.alfresco.repo.domain.qname.QNameDAO;
import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.namespace.QName;
import org.alfresco.util.Pair;
@@ -44,8 +44,8 @@ public class RMv22DODModelSeparationModulePatch extends AbstractModulePatch
/** query batch size */
private static final long BATCH_SIZE = 100000L;
/** QName DAO */
private QNameDAO qnameDAO;
/** indicates whether we convert to a standard file plan or not */
private boolean convertToStandardFilePlan = false;
/** Patch DAO */
private PatchDAO patchDAO;
@@ -61,16 +61,18 @@ public class RMv22DODModelSeparationModulePatch extends AbstractModulePatch
DOD5015Model.PROP_PUBLICATION_DATE,
DOD5015Model.PROP_MEDIA_TYPE,
DOD5015Model.PROP_FORMAT,
DOD5015Model.PROP_DATE_RECEIVED
DOD5015Model.PROP_DATE_RECEIVED,
DOD5015Model.PROP_ADDRESS,
DOD5015Model.PROP_OTHER_ADDRESS
};
/**
* @param qnameDAO QName DAO
* @param convertToStandardFilePlan convert to standard file if true, false otherwise
*/
public void setQnameDAO(QNameDAO qnameDAO)
public void setConvertToStandardFilePlan(boolean convertToStandardFilePlan)
{
this.qnameDAO = qnameDAO;
}
this.convertToStandardFilePlan = convertToStandardFilePlan;
}
/**
* @param patchDAO patch DAO
@@ -94,50 +96,73 @@ public class RMv22DODModelSeparationModulePatch extends AbstractModulePatch
@Override
public void applyInternal()
{
Long maxNodeId = patchDAO.getMaxAdmNodeID();
long recordCount = patchDAO.getCountNodesWithAspects(Collections.singleton(ASPECT_RECORD));
if (LOGGER.isDebugEnabled())
{
LOGGER.debug(" ... updating " + recordCount + " records");
}
if (!convertToStandardFilePlan)
{
Long maxNodeId = patchDAO.getMaxAdmNodeID();
long recordCount = patchDAO.getCountNodesWithAspects(Collections.singleton(ASPECT_RECORD));
if (LOGGER.isDebugEnabled())
{
LOGGER.debug(" ... updating " + recordCount + " records in batches of " + BATCH_SIZE);
}
// apply the DOD record aspect to all exiting records
int completed = 0;
Pair<Long, QName> recordAspect = qnameDAO.getQName(ASPECT_RECORD);
if (recordAspect != null)
{
for (Long i = 0L; i < maxNodeId; i+=BATCH_SIZE)
{
List<Long> nodeIds = patchDAO.getNodesByAspectQNameId(recordAspect.getFirst(), i, i + BATCH_SIZE);
for (Long nodeId : nodeIds)
{
// get the records properties
Map<QName, Serializable> properties = nodeDAO.getNodeProperties(nodeId);
// apply the DOD record aspect to all exiting records
int completed = 0;
for (Long i = 0L; i < maxNodeId; i+=BATCH_SIZE)
{
final Long finali = i;
Integer batchCount = transactionService.getRetryingTransactionHelper().doInTransaction(new RetryingTransactionCallback<Integer>()
{
int batchCount = 0;
for (QName qname : qnames)
{
// if the record has any of the moved properties
QName origional = QName.createQName(RecordsManagementModel.RM_URI, qname.getLocalName());
if (properties.containsKey(origional))
{
// move the property value
Serializable value = properties.get(origional);
properties.put(qname, value);
properties.remove(origional);
}
}
public Integer execute() throws Throwable
{
nodeDAO.getNodesWithAspects(Collections.singleton(ASPECT_RECORD), finali, finali + BATCH_SIZE, new NodeDAO.NodeRefQueryCallback()
{
public boolean handle(Pair<Long, NodeRef> nodePair)
{
// get the records properties
Map<QName, Serializable> properties = nodeDAO.getNodeProperties(nodePair.getFirst());
boolean changed = false;
// set properties and add aspect
nodeDAO.setNodeProperties(nodeId, properties);
nodeDAO.addNodeAspects(nodeId, Collections.singleton(DOD5015Model.ASPECT_DOD_5015_RECORD));
}
for (QName qname : qnames)
{
// if the record has any of the moved properties
QName origional = QName.createQName(RecordsManagementModel.RM_URI, qname.getLocalName());
if (properties.containsKey(origional))
{
// move the property value
Serializable value = properties.get(origional);
properties.put(qname, value);
properties.remove(origional);
changed = true;
}
}
completed += completed + nodeIds.size();
if (LOGGER.isDebugEnabled())
{
LOGGER.debug(" ... completed " + completed + " of " + recordCount);
}
}
}
// set properties and add aspect
if (changed)
{
nodeDAO.setNodeProperties(nodePair.getFirst(), properties);
}
nodeDAO.addNodeAspects(nodePair.getFirst(), Collections.singleton(DOD5015Model.ASPECT_DOD_5015_RECORD));
batchCount ++;
return true;
}
});
return batchCount;
}
} , false, true);
if (batchCount != 0)
{
completed = completed + batchCount;
if (LOGGER.isDebugEnabled())
{
LOGGER.debug(" ... completed " + completed + " of " + recordCount);
}
}
}
}
}
}