diff --git a/config/alfresco/core-services-context.xml b/config/alfresco/core-services-context.xml index e7089f4da4..978c51393b 100644 --- a/config/alfresco/core-services-context.xml +++ b/config/alfresco/core-services-context.xml @@ -476,6 +476,9 @@ + + + diff --git a/config/alfresco/messages/version-service.properties b/config/alfresco/messages/version-service.properties index 1593f6b295..d253d676a1 100644 --- a/config/alfresco/messages/version-service.properties +++ b/config/alfresco/messages/version-service.properties @@ -9,7 +9,7 @@ version_service.err_revert_mismatch=The version provided to revert to does not c version_service.migration.patch.noop=Nothing to do (no version histories found in old version store) -version_service.migration.patch.complete=Completed migration of {0} old version histories (to new version store) in {1} secs +version_service.migration.patch.complete=Completed migration of {0} (out of {1}) old version histories (to new version store) in {2} secs version_service.migration.patch.warn.skip1=Skipped migration of {0} old version histories (migrate failed) version_service.migration.patch.warn.skip2=Skipped migration of {0} old version histories (already migrated) diff --git a/source/java/org/alfresco/repo/version/VersionMigrator.java b/source/java/org/alfresco/repo/version/VersionMigrator.java index 7ac61c1146..13b99de918 100644 --- a/source/java/org/alfresco/repo/version/VersionMigrator.java +++ b/source/java/org/alfresco/repo/version/VersionMigrator.java @@ -68,7 +68,7 @@ public class VersionMigrator /** track completion * */ int percentComplete; - + /** start time * */ long startTime; @@ -146,7 +146,7 @@ public class VersionMigrator { logger.trace("migrateVersionHistory: oldVersionHistoryRef = " + oldVHNodeRef); } - + VersionHistory vh = v1BuildVersionHistory(oldVHNodeRef, versionedNodeRef); // create new version history node @@ -207,7 +207,7 @@ public class VersionMigrator Date versionModified = (Date)dbNodeService.getProperty(VersionUtil.convertNodeRef(frozenStateNodeRef), ContentModel.PROP_MODIFIED); String versionModifier = (String)dbNodeService.getProperty(VersionUtil.convertNodeRef(frozenStateNodeRef), ContentModel.PROP_MODIFIER); Date versionAccessed = (Date)dbNodeService.getProperty(VersionUtil.convertNodeRef(frozenStateNodeRef), ContentModel.PROP_ACCESSED); - + Map versionMetaDataProperties = version1Service.getVersionMetaData(VersionUtil.convertNodeRef(frozenStateNodeRef)); // Create the node details @@ -302,7 +302,9 @@ public class VersionMigrator public int migrateVersions(final int batchSize, final boolean deleteImmediately) { final NodeRef oldRootNodeRef = dbNodeService.getRootNode(VersionMigrator.VERSION_STORE_REF_OLD); - + final NodeRef newRootNodeRef = dbNodeService.getRootNode(VersionMigrator.VERSION_STORE_REF_NEW); + + long splitTime = System.currentTimeMillis(); final List childAssocRefs = getVersionHistories(oldRootNodeRef); int toDo = childAssocRefs.size(); @@ -313,11 +315,23 @@ public class VersionMigrator return 0; } - if (logger.isDebugEnabled()) + if (logger.isInfoEnabled()) { - logger.debug("Found "+toDo+" version histories in old version store"); + logger.info("Found "+childAssocRefs.size()+" version histories in old version store (in "+((System.currentTimeMillis()-splitTime)/1000)+" secs)"); } + splitTime = System.currentTimeMillis(); + final List newChildAssocRefs = getVersionHistories(newRootNodeRef); + final boolean firstMigration = (newChildAssocRefs.size() == 0); + + if (logger.isInfoEnabled()) + { + if (! firstMigration) + { + logger.warn("This is not the first migration attempt. Found "+newChildAssocRefs.size()+" version histories in new version store (in "+((System.currentTimeMillis()-splitTime)/1000)+" secs)"); + } + } + // note: assumes patch runs before cleanup starts startTime = System.currentTimeMillis(); percentComplete = 0; @@ -333,7 +347,6 @@ public class VersionMigrator try { - int batchCount = 0; int totalCount = 0; final List tmpBatch = new ArrayList(batchSize); @@ -343,80 +356,89 @@ public class VersionMigrator reportProgress(MSG_PATCH_PROGRESS, toDo, totalCount); totalCount++; - if (((String)dbNodeService.getProperty(childAssocRef.getChildRef(), ContentModel.PROP_NAME)).startsWith(VersionMigrator.PREFIX_MIGRATED)) + // short-cut if first migration + if (!firstMigration) { - // skip - already migrated - alreadyMigratedCount++; - continue; + if (isMigrated(childAssocRef)) + { + // skip - already migrated + alreadyMigratedCount++; + continue; + } } - if (batchCount < batchSize) + if (tmpBatch.size() < batchSize) { tmpBatch.add(childAssocRef.getChildRef()); - batchCount++; } - if ((batchCount == batchSize) || (totalCount == childAssocRefs.size())) + if ((tmpBatch.size() == batchSize) || (totalCount == childAssocRefs.size())) { while (tmpBatch.size() != 0) { txHelper.setMaxRetries(1); - NodeRef failed = txHelper.doInTransaction(new RetryingTransactionCallback() + try { - public NodeRef execute() throws Throwable + txHelper.doInTransaction(new RetryingTransactionCallback() { - if (logger.isTraceEnabled()) + public NodeRef execute() throws Throwable { - logger.trace("Attempt to migrate batch of "+tmpBatch.size()+" version histories"); - } - - long startTime = System.currentTimeMillis(); - - for (NodeRef oldVHNodeRef : tmpBatch) - { - try + if (logger.isTraceEnabled()) { - NodeRef versionedNodeRef = v1GetVersionedNodeRef(oldVHNodeRef); - migrateVersionHistory(oldVHNodeRef, versionedNodeRef); - - if (deleteImmediately) + logger.trace("Attempt to migrate batch of "+tmpBatch.size()+" version histories"); + } + + long startTime = System.currentTimeMillis(); + + for (NodeRef oldVHNodeRef : tmpBatch) + { + try { - // delete old version history node - v1DeleteVersionHistory(oldVHNodeRef); + NodeRef versionedNodeRef = v1GetVersionedNodeRef(oldVHNodeRef); + migrateVersionHistory(oldVHNodeRef, versionedNodeRef); + + if (deleteImmediately) + { + // delete old version history node + v1DeleteVersionHistory(oldVHNodeRef); + } + else + { + // mark old version history node for later cleanup + v1MarkVersionHistory(oldVHNodeRef); + } } - else + catch (Throwable t) { - // mark old version history node for later cleanup - v1MarkVersionHistory(oldVHNodeRef); - } + logger.error("Skipping migration of: " + oldVHNodeRef, t); + throw t; + } } - catch (Throwable t) + + if (logger.isDebugEnabled()) { - logger.error("Skipping migration of: " + oldVHNodeRef, t); - return oldVHNodeRef; + logger.debug("Migrated batch of "+tmpBatch.size()+" version histories in "+(System.currentTimeMillis()-startTime)+ " ms"); } + + return null; } - - if (logger.isDebugEnabled()) - { - logger.debug("Migrated batch of "+tmpBatch.size()+" version histories in "+(System.currentTimeMillis()-startTime)+ " ms"); - } - - return null; - } - }, false, true); - - if (failed != null) - { - tmpBatch.remove(failed); // retry batch without the failed node - failCount++; + }, false, true); + + // batch successful + vhCount = vhCount + tmpBatch.size(); + tmpBatch.clear(); } - else + catch (Throwable t) { - vhCount = vhCount + tmpBatch.size(); + // TODO if batchSize > 1 then could switch into batchSize=1 mode, and re-try one-by-one + // in theory, could fail on commit (although integrity checks are disabled by default) hence don't know which nodes failed + + logger.error("Skipping migration of batch size ("+tmpBatch.size()+"): "+t); + + // batch failed + failCount = failCount + tmpBatch.size(); tmpBatch.clear(); - batchCount = 0; } } } @@ -427,7 +449,7 @@ public class VersionMigrator MLPropertyInterceptor.setMLAware(wasMLAware); SessionSizeResourceManager.setEnableInTransaction(); } - + if (failCount > 0) { logger.warn(I18NUtil.getMessage(MSG_PATCH_SKIP1, failCount)); @@ -437,8 +459,17 @@ public class VersionMigrator logger.warn(I18NUtil.getMessage(MSG_PATCH_SKIP2, alreadyMigratedCount)); } - logger.info(I18NUtil.getMessage(MSG_PATCH_COMPLETE, vhCount, ((System.currentTimeMillis()-startTime)/1000))); + toDo = toDo - alreadyMigratedCount; + if (vhCount != toDo) + { + logger.warn(I18NUtil.getMessage(MSG_PATCH_COMPLETE, vhCount, toDo, ((System.currentTimeMillis()-startTime)/1000))); + } + else + { + logger.info(I18NUtil.getMessage(MSG_PATCH_COMPLETE, vhCount, toDo, ((System.currentTimeMillis()-startTime)/1000))); + } + return vhCount; } @@ -476,7 +507,7 @@ public class VersionMigrator int deletedCount = 0; int failCount = 0; int notMigratedCount = 0; - + boolean wasMLAware = MLPropertyInterceptor.setMLAware(true); SessionSizeResourceManager.setDisableInTransaction(); @@ -492,8 +523,8 @@ public class VersionMigrator reportProgress(MSG_DELETE_PROGRESS, toDo, totalCount); totalCount++; - if (((String)dbNodeService.getProperty(childAssocRef.getChildRef(), ContentModel.PROP_NAME)).startsWith(VersionMigrator.PREFIX_MIGRATED)) - { + if (isMigrated(childAssocRef)) + { if (batchCount < batchSize) { tmpBatch.add(childAssocRef.getChildRef()); @@ -596,6 +627,11 @@ public class VersionMigrator } } + protected boolean isMigrated(ChildAssociationRef vhChildAssocRef) + { + return (((String)dbNodeService.getProperty(vhChildAssocRef.getChildRef(), ContentModel.PROP_NAME)).startsWith(VersionMigrator.PREFIX_MIGRATED)); + } + /** * Support to report % completion and estimated completion time. * @@ -612,20 +648,20 @@ public class VersionMigrator { int previous = percentComplete; percentComplete = (int) (currentInteration * 100l / estimatedTotal); - + if (percentComplete < 100) { // conditional report long currentTime = System.currentTimeMillis(); long timeSoFar = currentTime - startTime; long timeRemaining = timeSoFar * (100 - percentComplete) / percentComplete; - + int report = -1; - + if (timeRemaining > 60000) { int reportInterval = getreportingInterval(timeSoFar, timeRemaining); - + for (int i = previous + 1; i <= percentComplete; i++) { if (i % reportInterval == 0) diff --git a/source/java/org/alfresco/repo/version/VersionMigratorTest.java b/source/java/org/alfresco/repo/version/VersionMigratorTest.java index 484b6200de..74c3a22013 100644 --- a/source/java/org/alfresco/repo/version/VersionMigratorTest.java +++ b/source/java/org/alfresco/repo/version/VersionMigratorTest.java @@ -31,7 +31,10 @@ import java.util.Map; import java.util.Set; import org.alfresco.model.ContentModel; +import org.alfresco.repo.node.integrity.IntegrityChecker; import org.alfresco.repo.policy.PolicyComponent; +import org.alfresco.repo.transaction.RetryingTransactionHelper; +import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; import org.alfresco.repo.version.common.counter.VersionCounterService; import org.alfresco.service.cmr.coci.CheckOutCheckInService; import org.alfresco.service.cmr.dictionary.DictionaryService; @@ -41,6 +44,7 @@ import org.alfresco.service.cmr.repository.NodeService; import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.service.cmr.version.Version; import org.alfresco.service.cmr.version.VersionHistory; +import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -62,6 +66,7 @@ public class VersionMigratorTest extends BaseVersionStoreTest protected DictionaryService dictionaryService; protected CheckOutCheckInService cociService; protected VersionCounterService versionCounterService; + protected IntegrityChecker integrityChecker; public VersionMigratorTest() { @@ -81,6 +86,8 @@ public class VersionMigratorTest extends BaseVersionStoreTest this.cociService = (CheckOutCheckInService)applicationContext.getBean("CheckoutCheckinService"); this.versionCounterService = (VersionCounterService)applicationContext.getBean("versionCounterService"); + this.integrityChecker = (IntegrityChecker)applicationContext.getBean("integrityChecker"); + // Version1Service is used to create the version nodes in Version1Store (workspace://lightWeightVersionStore) version1Service.setDbNodeService(dbNodeService); version1Service.setNodeService(dbNodeService); @@ -125,7 +132,7 @@ public class VersionMigratorTest extends BaseVersionStoreTest Map oldVersionProps = versionNodeService.getProperties(oldVersion.getFrozenStateNodeRef()); logger.info("oldVersion props: " + oldVersion); - logger.info("oldVersion created: " + oldVersion.getCreatedDate() + " [" + oldVersion.getCreatedDate().getTime()+"]"); + logger.info("oldVersion created: " + oldVersion.getFrozenModifiedDate() + " [" + oldVersion.getFrozenModifiedDate().getTime()+"]"); logger.info("oldVersion props via versionNodeService: " + oldVersionProps); @@ -147,7 +154,7 @@ public class VersionMigratorTest extends BaseVersionStoreTest Version newVersion = vh2.getRootVersion(); logger.info("newVersion props: " + newVersion); - logger.info("newVersion created: " + newVersion.getCreatedDate() + " [" + newVersion.getCreatedDate().getTime()+"]"); + logger.info("newVersion created: " + newVersion.getFrozenModifiedDate() + " [" + newVersion.getFrozenModifiedDate().getTime()+"]"); // check new version - switch to new version service to do the check super.setVersionService(version2Service); @@ -260,6 +267,167 @@ public class VersionMigratorTest extends BaseVersionStoreTest logger.info("testMigrateMultipleVersions: Migrated from oldVHNodeRef = " + oldVHNodeRef + " to newVHNodeRef = " + newVHNodeRef); } + public void testMigrateMultipleNodesSuccessful() throws Exception + { + testMigrateMultipleNodes(false); + } + + public void test_ETHREEOH_2091() throws Exception + { + // test partial migration (with skipped nodes) + testMigrateMultipleNodes(true); + } + + /** + * Test migration of a multiple nodes (each with one version) + */ + private void testMigrateMultipleNodes(final boolean withSkip) + { + if (version2Service.useDeprecatedV1 == true) + { + logger.info("testMigrateOneVersion: skip"); + return; + } + + final int nodeCount = 5; + assert(nodeCount > 3); + + final NodeRef[] versionableNodes = new NodeRef[nodeCount]; + + setComplete(); + endTransaction(); + + RetryingTransactionHelper txHelper = transactionService.getRetryingTransactionHelper(); + + for (int i = 0; i < nodeCount; i++) + { + final int idx = i; + + txHelper.doInTransaction(new RetryingTransactionCallback() + { + public NodeRef execute() throws Throwable + { + NodeRef versionableNode = null; + if ((idx % 2) == 0) + { + versionableNode = createNewVersionableNode(); + } + else + { + versionableNode = createNewVersionableContentNode(true); + } + createVersion(versionableNode); + versionableNodes[idx] = versionableNode; + + return null; + } + }); + } + + setComplete(); + endTransaction(); + + txHelper.doInTransaction(new RetryingTransactionCallback() + { + public NodeRef execute() throws Throwable + { + // check old version histories + for (int i = 0; i< nodeCount; i++) + { + VersionHistory vh1 = version1Service.getVersionHistory(versionableNodes[i]); + assertNotNull(vh1); + assertEquals(1, vh1.getAllVersions().size()); + } + + return null; + } + }); + + setComplete(); + endTransaction(); + + if (withSkip) + { + // remove test model - those nodes should fail - currently all - add separate create ... + + // TODO ... + dictionaryDAO.removeModel(QName.createQName("http://www.alfresco.org/test/versionstorebasetest/1.0", "versionstorebasetestmodel")); + } + + txHelper = transactionService.getRetryingTransactionHelper(); + + txHelper.doInTransaction(new RetryingTransactionCallback() + { + public NodeRef execute() throws Throwable + { + // Migrate (and don't delete old version history) ! + versionMigrator.migrateVersions(1, false); + + return null; + } + }); + + setComplete(); + endTransaction(); + + txHelper.doInTransaction(new RetryingTransactionCallback() + { + public NodeRef execute() throws Throwable + { + // check new version histories + for (int i = 0; i < nodeCount; i++) + { + VersionHistory vh2 = version2Service.getVersionHistory(versionableNodes[i]); + + if (withSkip && ((i % 2) == 0)) + { + assertNull(vh2); + } + else + { + assertNotNull(vh2); + assertEquals(1, vh2.getAllVersions().size()); + } + } + + return null; + } + }); + } + + private NodeRef createNewVersionableContentNode(boolean versionable) + { + // Use this map to retrieve the versionable nodes in later tests + this.versionableNodes = new HashMap(); + + // Create node (this node has some content) + NodeRef nodeRef = this.dbNodeService.createNode( + rootNodeRef, + ContentModel.ASSOC_CHILDREN, + QName.createQName(NamespaceService.CONTENT_MODEL_1_0_URI, "myNode"), + ContentModel.TYPE_CONTENT, + this.nodeProperties).getChildRef(); + + if (versionable) + { + this.dbNodeService.addAspect(nodeRef, ContentModel.ASPECT_VERSIONABLE, new HashMap()); + } + + assertNotNull(nodeRef); + this.versionableNodes.put(nodeRef.getId(), nodeRef); + + // Add the content to the node + ContentWriter contentWriter = this.contentService.getWriter(nodeRef, ContentModel.PROP_CONTENT, true); + contentWriter.putContent(TEST_CONTENT); + + // Set author + Map authorProps = new HashMap(1, 1.0f); + authorProps.put(ContentModel.PROP_AUTHOR, "Charles Dickens"); + this.dbNodeService.addAspect(nodeRef, ContentModel.ASPECT_AUTHOR, authorProps); + + return nodeRef; + } + public void test_ETHREEOH_1540() throws Exception { // Create the node used for tests