From f8852e3631b249ac31f91b4637c74372e3a923f4 Mon Sep 17 00:00:00 2001 From: Andrew Hind Date: Sat, 20 Mar 2010 16:55:20 +0000 Subject: [PATCH] Merged V3.2 to HEAD 19432: Merged V3.1 to V3.2 19427: Merged V3.0 to V3.1 19423: Merged V2.2 to V3.0 19391: Fix for ALF-2076: AUTO does not work if a document has been added and deleted since the index backup 19419: V2.2 Build Fix 19421: Fix for ALF-2076: AUTO does not work if a document has been added and deleted since the index backup git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@19434 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261 --- .../node/index/AbstractReindexComponent.java | 42 +++---- .../index/FullIndexRecoveryComponent.java | 90 ++++++++++++-- .../index/FullIndexRecoveryComponentTest.java | 115 ++++++++++++++++++ .../node/index/IndexTransactionTracker.java | 7 +- 4 files changed, 216 insertions(+), 38 deletions(-) diff --git a/source/java/org/alfresco/repo/node/index/AbstractReindexComponent.java b/source/java/org/alfresco/repo/node/index/AbstractReindexComponent.java index 4400bcb9bd..fb26b2a1af 100644 --- a/source/java/org/alfresco/repo/node/index/AbstractReindexComponent.java +++ b/source/java/org/alfresco/repo/node/index/AbstractReindexComponent.java @@ -369,15 +369,16 @@ public abstract class AbstractReindexComponent implements IndexRecovery return storeRefs; } - protected InIndex isTxnIdPresentInIndex(long txnId) - { - Transaction txn = nodeDaoService.getTxnById(txnId); - if (txn == null) - { - return InIndex.YES; - } - return isTxnPresentInIndex(txn); - } +// Unused - comemnted out to make use clearer for isTxnPresentInIndex +// protected InIndex isTxnIdPresentInIndex(long txnId) +// { +// Transaction txn = nodeDaoService.getTxnById(txnId); +// if (txn == null) +// { +// return InIndex.YES; +// } +// return isTxnPresentInIndex(txn); +// } /** * Determines if a given transaction is definitely in the index or not. @@ -463,13 +464,16 @@ public abstract class AbstractReindexComponent implements IndexRecovery { // There are no updates or deletes and no entry in the indexes. // There are outdated nodes in the index. - result = InIndex.YES; + result = InIndex.INDETERMINATE; } else { // There were deleted nodes only. Check that all the deleted nodes were // removed from the index otherwise it is out of date. - result = InIndex.YES; + // If all nodes have been removed from the index then the result is that the index is OK + // ETWOTWO-1387 + // ALF-1989 - even if the nodes have not been found it is no good to use for AUTO index checking + result = InIndex.INDETERMINATE; for (StoreRef storeRef : storeRefs) { if (!haveNodesBeenRemovedFromIndex(storeRef, txn)) @@ -635,22 +639,6 @@ public abstract class AbstractReindexComponent implements IndexRecovery return !foundNodeRef; } - /** - * @return Returns false if any one of the transactions aren't in the index. - */ - protected boolean areTxnsInIndex(List txns) - { - for (Transaction txn : txns) - { - if (isTxnPresentInIndex(txn) == InIndex.NO) - { - // Missing txn - return false; - } - } - return true; - } - /** * Marker exception to neatly handle VM-driven termination of a reindex * diff --git a/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponent.java b/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponent.java index 3f2e6f2802..e9020d271d 100644 --- a/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponent.java +++ b/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponent.java @@ -27,6 +27,7 @@ import java.util.List; import org.springframework.extensions.surf.util.I18NUtil; import org.alfresco.model.ContentModel; import org.alfresco.repo.domain.Transaction; +import org.alfresco.repo.node.index.AbstractReindexComponent.InIndex; import org.alfresco.repo.node.index.IndexTransactionTracker.IndexTransactionTrackerListener; import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; import org.alfresco.service.cmr.repository.ChildAssociationRef; @@ -182,31 +183,56 @@ public class FullIndexRecoveryComponent extends AbstractReindexComponent transactionService.setAllowWrite(false); } - // Check that the first and last meaningful transactions are indexed - List startTxns = nodeDaoService.getTxnsByCommitTimeAscending( - Long.MIN_VALUE, Long.MAX_VALUE, 10, null, false); - boolean startAllPresent = areTxnsInIndex(startTxns); - List endTxns = nodeDaoService.getTxnsByCommitTimeDescending( - Long.MIN_VALUE, Long.MAX_VALUE, 10, null, false); - boolean endAllPresent = areTxnsInIndex(endTxns); + int startSample = 10; + InIndex startAllPresent; + do + { + // Check that the first and last meaningful transactions are indexed + List startTxns = nodeDaoService.getTxnsByCommitTimeAscending( + Long.MIN_VALUE, Long.MAX_VALUE, startSample, null, false); + startAllPresent = areTxnsInStartSample(startTxns); + startSample += 10; + if(startSample > 1000) + { + startAllPresent = InIndex.NO; + break; + } + } + while(startAllPresent == InIndex.INDETERMINATE); + + int endSample = 10; + InIndex endAllPresent; + do + { + List endTxns = nodeDaoService.getTxnsByCommitTimeDescending( + Long.MIN_VALUE, Long.MAX_VALUE, endSample, null, false); + endAllPresent = areAllTxnsInEndSample(endTxns); + endSample += 10; + if(endSample > 1000) + { + endAllPresent = InIndex.NO; + break; + } + } + while(endAllPresent == InIndex.INDETERMINATE); // check the level of cover required switch (recoveryMode) { case AUTO: - if (!startAllPresent) + if (startAllPresent == InIndex.NO) { // Initial transactions are missing - rebuild performFullRecovery(); } - else if (!endAllPresent) + else if (endAllPresent == InIndex.NO) { performPartialRecovery(); } break; case VALIDATE: // Check - if (!startAllPresent || !endAllPresent) + if ((startAllPresent == InIndex.NO) || (endAllPresent == InIndex.NO)) { // Index is out of date logger.warn(I18NUtil.getMessage(ERR_INDEX_OUT_OF_DATE)); @@ -225,6 +251,50 @@ public class FullIndexRecoveryComponent extends AbstractReindexComponent } + /** + * @return Returns false if any one of the transactions aren't in the index. + */ + protected InIndex areAllTxnsInEndSample(List txns) + { + int yesCount = 0; + for (Transaction txn : txns) + { + if (isTxnPresentInIndex(txn) == InIndex.NO) + { + // Missing txn + return InIndex.NO; + } + if (isTxnPresentInIndex(txn) == InIndex.YES) + { + yesCount++; + } + } + // Work around for TX that is written to the repo at start up .. must be more than one real add /update + if(yesCount > 1) + { + return InIndex.YES; + } + else + { + return InIndex.INDETERMINATE; + } + } + + protected InIndex areTxnsInStartSample(List txns) + { + InIndex current = InIndex.INDETERMINATE; + for (Transaction txn : txns) + { + current = isTxnPresentInIndex(txn); + if (current == InIndex.NO) + { + // Missing txn + return InIndex.NO; + } + } + return current; + } + private void performPartialRecovery() { // Log the AUTO recovery diff --git a/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponentTest.java b/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponentTest.java index a51c078e5d..43a80f08e9 100644 --- a/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponentTest.java +++ b/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponentTest.java @@ -18,8 +18,27 @@ */ package org.alfresco.repo.node.index; +import java.util.List; +import java.util.Locale; + +import javax.transaction.UserTransaction; + import junit.framework.TestCase; +import org.alfresco.i18n.I18NUtil; +import org.alfresco.model.ContentModel; +import org.alfresco.repo.dictionary.NamespaceDAOImpl; +import org.alfresco.repo.domain.Transaction; +import org.alfresco.repo.node.db.NodeDaoService; +import org.alfresco.repo.node.index.AbstractReindexComponent.InIndex; +import org.alfresco.repo.security.authentication.AuthenticationComponent; +import org.alfresco.repo.transaction.RetryingTransactionHelper; +import org.alfresco.service.ServiceRegistry; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.NodeService; +import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.namespace.QName; +import org.alfresco.service.transaction.TransactionService; import org.alfresco.util.ApplicationContextHelper; import org.springframework.context.ApplicationContext; @@ -35,10 +54,37 @@ public class FullIndexRecoveryComponentTest extends TestCase private FullIndexRecoveryComponent indexRecoverer; private AVMFullIndexRecoveryComponent avmIndexRecoveryComponent; + + private NodeService nodeService; + + private NodeRef rootNodeRef; + + private TransactionService transactionService; + + private RetryingTransactionHelper retryingTransactionHelper; + + private AuthenticationComponent authenticationComponent; + + private UserTransaction testTX; + + private NodeDaoService nodeDaoService; + public void setUp() throws Exception { indexRecoverer = (FullIndexRecoveryComponent) ctx.getBean("indexRecoveryComponent"); avmIndexRecoveryComponent = (AVMFullIndexRecoveryComponent) ctx.getBean("avmIndexRecoveryComponent"); + nodeService = (NodeService) ctx.getBean("nodeService"); + transactionService = (TransactionService) ctx.getBean("transactionComponent"); + retryingTransactionHelper = (RetryingTransactionHelper) ctx.getBean("retryingTransactionHelper"); + authenticationComponent = (AuthenticationComponent) ctx.getBean("authenticationComponent"); + nodeDaoService = (NodeDaoService) ctx.getBean("nodeDaoServiceImpl"); + + testTX = transactionService.getUserTransaction(); + testTX.begin(); + this.authenticationComponent.setSystemUserAsCurrentUser(); + + + } @@ -47,6 +93,75 @@ public class FullIndexRecoveryComponentTest extends TestCase } + public void XtestDeletionReporting() throws Exception + { + StoreRef storeRef = nodeService.createStore(StoreRef.PROTOCOL_WORKSPACE, "Test_" + System.currentTimeMillis()); + rootNodeRef = nodeService.getRootNode(storeRef); + + NodeRef folder = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}folder"), ContentModel.TYPE_FOLDER).getChildRef(); + + testTX.commit(); + testTX = transactionService.getUserTransaction(); + testTX.begin(); + + NodeRef[] refs = new NodeRef[20]; + for(int i = 0; i < refs.length; i++) + { + refs[i] = nodeService.createNode(folder, ContentModel.ASSOC_CONTAINS, QName.createQName("{namespace}file"+i), ContentModel.TYPE_CONTENT).getChildRef(); + } + + testTX.commit(); + testTX = transactionService.getUserTransaction(); + testTX.begin(); + this.authenticationComponent.setSystemUserAsCurrentUser(); + for(int i = 0; i < refs.length; i++) + { + nodeService.deleteNode(refs[i]); + testTX.commit(); + testTX = transactionService.getUserTransaction(); + testTX.begin(); + } + + // The following test are important but take too long .... + +// List startTxns = nodeDaoService.getTxnsByCommitTimeAscending( +// Long.MIN_VALUE, Long.MAX_VALUE, 1, null, false); +// InIndex startAllPresent = indexRecoverer.areTxnsInStartSample(startTxns); +// assertEquals(InIndex.YES, startAllPresent); +// Long maxId = nodeDaoService.getMaxTxnIdByCommitTime(Long.MAX_VALUE); +// startTxns = nodeDaoService.getTxnsByCommitTimeAscending( +// Long.MIN_VALUE, Long.MAX_VALUE, maxId.intValue(), null, false); +// startAllPresent = indexRecoverer.areTxnsInStartSample(startTxns); +// assertEquals(InIndex.INDETERMINATE, startAllPresent); + +// for(int i = 0; i <= maxId.intValue(); i++) +// { +// System.out.println("TX "+i+" is "+indexRecoverer.isTxnPresentInIndex(nodeDaoService.getTxnById(i))); +// } + +// startTxns = nodeDaoService.getTxnsByCommitTimeAscending( +// Long.MIN_VALUE, Long.MAX_VALUE, startTxns.size() - 20, null, false); +// startAllPresent = indexRecoverer.areTxnsInStartSample(startTxns); +// assertEquals(InIndex.YES, startAllPresent); +// + + + List endTxns = nodeDaoService.getTxnsByCommitTimeDescending( + Long.MIN_VALUE, Long.MAX_VALUE, 20, null, false); + InIndex endAllPresent = indexRecoverer.areAllTxnsInEndSample(endTxns); + assertEquals(InIndex.INDETERMINATE, endAllPresent); + + endTxns = nodeDaoService.getTxnsByCommitTimeDescending( + Long.MIN_VALUE, Long.MAX_VALUE, 21, null, false); + endAllPresent = indexRecoverer.areAllTxnsInEndSample(endTxns); + assertEquals(InIndex.INDETERMINATE, endAllPresent); + + endTxns = nodeDaoService.getTxnsByCommitTimeDescending( + Long.MIN_VALUE, Long.MAX_VALUE, 22, null, false); + endAllPresent = indexRecoverer.areAllTxnsInEndSample(endTxns); + assertEquals(InIndex.YES, endAllPresent); + } + public synchronized void testReindexing() throws Exception { indexRecoverer.setRecoveryMode(FullIndexRecoveryComponent.RecoveryMode.FULL.name()); diff --git a/source/java/org/alfresco/repo/node/index/IndexTransactionTracker.java b/source/java/org/alfresco/repo/node/index/IndexTransactionTracker.java index 5f58b847de..2d6b828a70 100644 --- a/source/java/org/alfresco/repo/node/index/IndexTransactionTracker.java +++ b/source/java/org/alfresco/repo/node/index/IndexTransactionTracker.java @@ -456,11 +456,16 @@ found: case YES: fromTimeInclusive = txnCommitTime; break found; + case INDETERMINATE: + // If we hit an indeterminate transaction we go back a small amount to try and hit something definitive before a bigger step back + firstWasInIndex = false; + toTimeExclusive = txnCommitTime - 1000; + continue; default: firstWasInIndex = false; // Look further back in time. Step back by 60 seconds each time, increasing // the step by 10% each iteration. - // Don't step back by more than a day + // Don't step back by more than an hour long decrement = Math.min(ONE_HOUR_MS, (long) (60000.0D * stepFactor)); toTimeExclusive = txnCommitTime - decrement; stepFactor *= 1.1D;