diff --git a/source/java/org/alfresco/repo/node/index/AbstractReindexComponent.java b/source/java/org/alfresco/repo/node/index/AbstractReindexComponent.java
index 4400bcb9bd..fb26b2a1af 100644
--- a/source/java/org/alfresco/repo/node/index/AbstractReindexComponent.java
+++ b/source/java/org/alfresco/repo/node/index/AbstractReindexComponent.java
@@ -369,15 +369,16 @@ public abstract class AbstractReindexComponent implements IndexRecovery
return storeRefs;
}
- protected InIndex isTxnIdPresentInIndex(long txnId)
- {
- Transaction txn = nodeDaoService.getTxnById(txnId);
- if (txn == null)
- {
- return InIndex.YES;
- }
- return isTxnPresentInIndex(txn);
- }
+// Unused - comemnted out to make use clearer for isTxnPresentInIndex
+// protected InIndex isTxnIdPresentInIndex(long txnId)
+// {
+// Transaction txn = nodeDaoService.getTxnById(txnId);
+// if (txn == null)
+// {
+// return InIndex.YES;
+// }
+// return isTxnPresentInIndex(txn);
+// }
/**
* Determines if a given transaction is definitely in the index or not.
@@ -463,13 +464,16 @@ public abstract class AbstractReindexComponent implements IndexRecovery
{
// There are no updates or deletes and no entry in the indexes.
// There are outdated nodes in the index.
- result = InIndex.YES;
+ result = InIndex.INDETERMINATE;
}
else
{
// There were deleted nodes only. Check that all the deleted nodes were
// removed from the index otherwise it is out of date.
- result = InIndex.YES;
+ // If all nodes have been removed from the index then the result is that the index is OK
+ // ETWOTWO-1387
+ // ALF-1989 - even if the nodes have not been found it is no good to use for AUTO index checking
+ result = InIndex.INDETERMINATE;
for (StoreRef storeRef : storeRefs)
{
if (!haveNodesBeenRemovedFromIndex(storeRef, txn))
@@ -635,22 +639,6 @@ public abstract class AbstractReindexComponent implements IndexRecovery
return !foundNodeRef;
}
- /**
- * @return Returns false if any one of the transactions aren't in the index.
- */
- protected boolean areTxnsInIndex(List txns)
- {
- for (Transaction txn : txns)
- {
- if (isTxnPresentInIndex(txn) == InIndex.NO)
- {
- // Missing txn
- return false;
- }
- }
- return true;
- }
-
/**
* Marker exception to neatly handle VM-driven termination of a reindex
*
diff --git a/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponent.java b/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponent.java
index 3f2e6f2802..e9020d271d 100644
--- a/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponent.java
+++ b/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponent.java
@@ -27,6 +27,7 @@ import java.util.List;
import org.springframework.extensions.surf.util.I18NUtil;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.domain.Transaction;
+import org.alfresco.repo.node.index.AbstractReindexComponent.InIndex;
import org.alfresco.repo.node.index.IndexTransactionTracker.IndexTransactionTrackerListener;
import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
import org.alfresco.service.cmr.repository.ChildAssociationRef;
@@ -182,31 +183,56 @@ public class FullIndexRecoveryComponent extends AbstractReindexComponent
transactionService.setAllowWrite(false);
}
- // Check that the first and last meaningful transactions are indexed
- List startTxns = nodeDaoService.getTxnsByCommitTimeAscending(
- Long.MIN_VALUE, Long.MAX_VALUE, 10, null, false);
- boolean startAllPresent = areTxnsInIndex(startTxns);
- List endTxns = nodeDaoService.getTxnsByCommitTimeDescending(
- Long.MIN_VALUE, Long.MAX_VALUE, 10, null, false);
- boolean endAllPresent = areTxnsInIndex(endTxns);
+ int startSample = 10;
+ InIndex startAllPresent;
+ do
+ {
+ // Check that the first and last meaningful transactions are indexed
+ List startTxns = nodeDaoService.getTxnsByCommitTimeAscending(
+ Long.MIN_VALUE, Long.MAX_VALUE, startSample, null, false);
+ startAllPresent = areTxnsInStartSample(startTxns);
+ startSample += 10;
+ if(startSample > 1000)
+ {
+ startAllPresent = InIndex.NO;
+ break;
+ }
+ }
+ while(startAllPresent == InIndex.INDETERMINATE);
+
+ int endSample = 10;
+ InIndex endAllPresent;
+ do
+ {
+ List endTxns = nodeDaoService.getTxnsByCommitTimeDescending(
+ Long.MIN_VALUE, Long.MAX_VALUE, endSample, null, false);
+ endAllPresent = areAllTxnsInEndSample(endTxns);
+ endSample += 10;
+ if(endSample > 1000)
+ {
+ endAllPresent = InIndex.NO;
+ break;
+ }
+ }
+ while(endAllPresent == InIndex.INDETERMINATE);
// check the level of cover required
switch (recoveryMode)
{
case AUTO:
- if (!startAllPresent)
+ if (startAllPresent == InIndex.NO)
{
// Initial transactions are missing - rebuild
performFullRecovery();
}
- else if (!endAllPresent)
+ else if (endAllPresent == InIndex.NO)
{
performPartialRecovery();
}
break;
case VALIDATE:
// Check
- if (!startAllPresent || !endAllPresent)
+ if ((startAllPresent == InIndex.NO) || (endAllPresent == InIndex.NO))
{
// Index is out of date
logger.warn(I18NUtil.getMessage(ERR_INDEX_OUT_OF_DATE));
@@ -225,6 +251,50 @@ public class FullIndexRecoveryComponent extends AbstractReindexComponent
}
+ /**
+ * @return Returns false if any one of the transactions aren't in the index.
+ */
+ protected InIndex areAllTxnsInEndSample(List txns)
+ {
+ int yesCount = 0;
+ for (Transaction txn : txns)
+ {
+ if (isTxnPresentInIndex(txn) == InIndex.NO)
+ {
+ // Missing txn
+ return InIndex.NO;
+ }
+ if (isTxnPresentInIndex(txn) == InIndex.YES)
+ {
+ yesCount++;
+ }
+ }
+ // Work around for TX that is written to the repo at start up .. must be more than one real add /update
+ if(yesCount > 1)
+ {
+ return InIndex.YES;
+ }
+ else
+ {
+ return InIndex.INDETERMINATE;
+ }
+ }
+
+ protected InIndex areTxnsInStartSample(List txns)
+ {
+ InIndex current = InIndex.INDETERMINATE;
+ for (Transaction txn : txns)
+ {
+ current = isTxnPresentInIndex(txn);
+ if (current == InIndex.NO)
+ {
+ // Missing txn
+ return InIndex.NO;
+ }
+ }
+ return current;
+ }
+
private void performPartialRecovery()
{
// Log the AUTO recovery
diff --git a/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponentTest.java b/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponentTest.java
index a51c078e5d..43a80f08e9 100644
--- a/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponentTest.java
+++ b/source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponentTest.java
@@ -18,8 +18,27 @@
*/
package org.alfresco.repo.node.index;
+import java.util.List;
+import java.util.Locale;
+
+import javax.transaction.UserTransaction;
+
import junit.framework.TestCase;
+import org.alfresco.i18n.I18NUtil;
+import org.alfresco.model.ContentModel;
+import org.alfresco.repo.dictionary.NamespaceDAOImpl;
+import org.alfresco.repo.domain.Transaction;
+import org.alfresco.repo.node.db.NodeDaoService;
+import org.alfresco.repo.node.index.AbstractReindexComponent.InIndex;
+import org.alfresco.repo.security.authentication.AuthenticationComponent;
+import org.alfresco.repo.transaction.RetryingTransactionHelper;
+import org.alfresco.service.ServiceRegistry;
+import org.alfresco.service.cmr.repository.NodeRef;
+import org.alfresco.service.cmr.repository.NodeService;
+import org.alfresco.service.cmr.repository.StoreRef;
+import org.alfresco.service.namespace.QName;
+import org.alfresco.service.transaction.TransactionService;
import org.alfresco.util.ApplicationContextHelper;
import org.springframework.context.ApplicationContext;
@@ -35,10 +54,37 @@ public class FullIndexRecoveryComponentTest extends TestCase
private FullIndexRecoveryComponent indexRecoverer;
private AVMFullIndexRecoveryComponent avmIndexRecoveryComponent;
+
+ private NodeService nodeService;
+
+ private NodeRef rootNodeRef;
+
+ private TransactionService transactionService;
+
+ private RetryingTransactionHelper retryingTransactionHelper;
+
+ private AuthenticationComponent authenticationComponent;
+
+ private UserTransaction testTX;
+
+ private NodeDaoService nodeDaoService;
+
public void setUp() throws Exception
{
indexRecoverer = (FullIndexRecoveryComponent) ctx.getBean("indexRecoveryComponent");
avmIndexRecoveryComponent = (AVMFullIndexRecoveryComponent) ctx.getBean("avmIndexRecoveryComponent");
+ nodeService = (NodeService) ctx.getBean("nodeService");
+ transactionService = (TransactionService) ctx.getBean("transactionComponent");
+ retryingTransactionHelper = (RetryingTransactionHelper) ctx.getBean("retryingTransactionHelper");
+ authenticationComponent = (AuthenticationComponent) ctx.getBean("authenticationComponent");
+ nodeDaoService = (NodeDaoService) ctx.getBean("nodeDaoServiceImpl");
+
+ testTX = transactionService.getUserTransaction();
+ testTX.begin();
+ this.authenticationComponent.setSystemUserAsCurrentUser();
+
+
+
}
@@ -47,6 +93,75 @@ public class FullIndexRecoveryComponentTest extends TestCase
}
+ public void XtestDeletionReporting() throws Exception
+ {
+ StoreRef storeRef = nodeService.createStore(StoreRef.PROTOCOL_WORKSPACE, "Test_" + System.currentTimeMillis());
+ rootNodeRef = nodeService.getRootNode(storeRef);
+
+ NodeRef folder = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}folder"), ContentModel.TYPE_FOLDER).getChildRef();
+
+ testTX.commit();
+ testTX = transactionService.getUserTransaction();
+ testTX.begin();
+
+ NodeRef[] refs = new NodeRef[20];
+ for(int i = 0; i < refs.length; i++)
+ {
+ refs[i] = nodeService.createNode(folder, ContentModel.ASSOC_CONTAINS, QName.createQName("{namespace}file"+i), ContentModel.TYPE_CONTENT).getChildRef();
+ }
+
+ testTX.commit();
+ testTX = transactionService.getUserTransaction();
+ testTX.begin();
+ this.authenticationComponent.setSystemUserAsCurrentUser();
+ for(int i = 0; i < refs.length; i++)
+ {
+ nodeService.deleteNode(refs[i]);
+ testTX.commit();
+ testTX = transactionService.getUserTransaction();
+ testTX.begin();
+ }
+
+ // The following test are important but take too long ....
+
+// List startTxns = nodeDaoService.getTxnsByCommitTimeAscending(
+// Long.MIN_VALUE, Long.MAX_VALUE, 1, null, false);
+// InIndex startAllPresent = indexRecoverer.areTxnsInStartSample(startTxns);
+// assertEquals(InIndex.YES, startAllPresent);
+// Long maxId = nodeDaoService.getMaxTxnIdByCommitTime(Long.MAX_VALUE);
+// startTxns = nodeDaoService.getTxnsByCommitTimeAscending(
+// Long.MIN_VALUE, Long.MAX_VALUE, maxId.intValue(), null, false);
+// startAllPresent = indexRecoverer.areTxnsInStartSample(startTxns);
+// assertEquals(InIndex.INDETERMINATE, startAllPresent);
+
+// for(int i = 0; i <= maxId.intValue(); i++)
+// {
+// System.out.println("TX "+i+" is "+indexRecoverer.isTxnPresentInIndex(nodeDaoService.getTxnById(i)));
+// }
+
+// startTxns = nodeDaoService.getTxnsByCommitTimeAscending(
+// Long.MIN_VALUE, Long.MAX_VALUE, startTxns.size() - 20, null, false);
+// startAllPresent = indexRecoverer.areTxnsInStartSample(startTxns);
+// assertEquals(InIndex.YES, startAllPresent);
+//
+
+
+ List endTxns = nodeDaoService.getTxnsByCommitTimeDescending(
+ Long.MIN_VALUE, Long.MAX_VALUE, 20, null, false);
+ InIndex endAllPresent = indexRecoverer.areAllTxnsInEndSample(endTxns);
+ assertEquals(InIndex.INDETERMINATE, endAllPresent);
+
+ endTxns = nodeDaoService.getTxnsByCommitTimeDescending(
+ Long.MIN_VALUE, Long.MAX_VALUE, 21, null, false);
+ endAllPresent = indexRecoverer.areAllTxnsInEndSample(endTxns);
+ assertEquals(InIndex.INDETERMINATE, endAllPresent);
+
+ endTxns = nodeDaoService.getTxnsByCommitTimeDescending(
+ Long.MIN_VALUE, Long.MAX_VALUE, 22, null, false);
+ endAllPresent = indexRecoverer.areAllTxnsInEndSample(endTxns);
+ assertEquals(InIndex.YES, endAllPresent);
+ }
+
public synchronized void testReindexing() throws Exception
{
indexRecoverer.setRecoveryMode(FullIndexRecoveryComponent.RecoveryMode.FULL.name());
diff --git a/source/java/org/alfresco/repo/node/index/IndexTransactionTracker.java b/source/java/org/alfresco/repo/node/index/IndexTransactionTracker.java
index 5f58b847de..2d6b828a70 100644
--- a/source/java/org/alfresco/repo/node/index/IndexTransactionTracker.java
+++ b/source/java/org/alfresco/repo/node/index/IndexTransactionTracker.java
@@ -456,11 +456,16 @@ found:
case YES:
fromTimeInclusive = txnCommitTime;
break found;
+ case INDETERMINATE:
+ // If we hit an indeterminate transaction we go back a small amount to try and hit something definitive before a bigger step back
+ firstWasInIndex = false;
+ toTimeExclusive = txnCommitTime - 1000;
+ continue;
default:
firstWasInIndex = false;
// Look further back in time. Step back by 60 seconds each time, increasing
// the step by 10% each iteration.
- // Don't step back by more than a day
+ // Don't step back by more than an hour
long decrement = Math.min(ONE_HOUR_MS, (long) (60000.0D * stepFactor));
toTimeExclusive = txnCommitTime - decrement;
stepFactor *= 1.1D;