Merged V3.2 to HEAD

19432: Merged V3.1 to V3.2
        19427: Merged V3.0 to V3.1
            19423: Merged V2.2 to V3.0
                19391: Fix for ALF-2076: AUTO does not work if a document has been added and deleted since the index backup
                19419: V2.2 Build Fix
                19421: Fix for ALF-2076: AUTO does not work if a document has been added and deleted since the index backup


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@19434 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/repo/node/index/AbstractReindexComponent.java

@@ -369,15 +369,16 @@ public abstract class AbstractReindexComponent implements IndexRecovery
         return storeRefs;
     }
     
-    protected InIndex isTxnIdPresentInIndex(long txnId)
+//    Unused - comemnted out to make use clearer for isTxnPresentInIndex
-    {
+//    protected InIndex isTxnIdPresentInIndex(long txnId)
-        Transaction txn = nodeDaoService.getTxnById(txnId);
+//    {
-        if (txn == null)
+//        Transaction txn = nodeDaoService.getTxnById(txnId);
-        {
+//        if (txn == null)
-            return InIndex.YES;
+//        {
-        }
+//            return InIndex.YES;
-        return isTxnPresentInIndex(txn);
+//        }
-    }
+//        return isTxnPresentInIndex(txn);
+//    }
     
     /**
      * Determines if a given transaction is definitely in the index or not.
@@ -463,13 +464,16 @@ public abstract class AbstractReindexComponent implements IndexRecovery
                 {
                     // There are no updates or deletes and no entry in the indexes.
                     // There are outdated nodes in the index.
-                    result = InIndex.YES;
+                    result = InIndex.INDETERMINATE;
                 }
                 else
                 {
                     // There were deleted nodes only.  Check that all the deleted nodes were
                     // removed from the index otherwise it is out of date.
-                    result = InIndex.YES;
+                    // If all nodes have been removed from the index then the result is that the index is OK
+                    // ETWOTWO-1387
+                    // ALF-1989 - even if the nodes have not been found it is no good to use for AUTO index checking 
+                    result = InIndex.INDETERMINATE;
                     for (StoreRef storeRef : storeRefs)
                     {
                         if (!haveNodesBeenRemovedFromIndex(storeRef, txn))
@@ -635,22 +639,6 @@ public abstract class AbstractReindexComponent implements IndexRecovery
         return !foundNodeRef;
     }
     
-    /**
-     * @return          Returns <tt>false</tt> if any one of the transactions aren't in the index.
-     */
-    protected boolean areTxnsInIndex(List<Transaction> txns)
-    {
-        for (Transaction txn : txns)
-        {
-            if (isTxnPresentInIndex(txn) == InIndex.NO)
-            {
-                // Missing txn
-                return false;
-            }
-        }
-        return true;
-    }
-    
     /**
      * Marker exception to neatly handle VM-driven termination of a reindex
      * 

source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponent.java

@@ -27,6 +27,7 @@ import java.util.List;
 import org.springframework.extensions.surf.util.I18NUtil;
 import org.alfresco.model.ContentModel;
 import org.alfresco.repo.domain.Transaction;
+import org.alfresco.repo.node.index.AbstractReindexComponent.InIndex;
 import org.alfresco.repo.node.index.IndexTransactionTracker.IndexTransactionTrackerListener;
 import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
 import org.alfresco.service.cmr.repository.ChildAssociationRef;
@@ -182,31 +183,56 @@ public class FullIndexRecoveryComponent extends AbstractReindexComponent
                 transactionService.setAllowWrite(false);
             }
             
-            // Check that the first and last meaningful transactions are indexed 
+            int startSample = 10;
-            List<Transaction> startTxns = nodeDaoService.getTxnsByCommitTimeAscending(
+            InIndex startAllPresent;
-                    Long.MIN_VALUE, Long.MAX_VALUE, 10, null, false);
+            do
-            boolean startAllPresent = areTxnsInIndex(startTxns);
+            {
-            List<Transaction> endTxns = nodeDaoService.getTxnsByCommitTimeDescending(
+                // Check that the first and last meaningful transactions are indexed 
-                    Long.MIN_VALUE, Long.MAX_VALUE, 10, null, false);
+                List<Transaction> startTxns = nodeDaoService.getTxnsByCommitTimeAscending(
-            boolean endAllPresent = areTxnsInIndex(endTxns);
+                        Long.MIN_VALUE, Long.MAX_VALUE, startSample, null, false);
+                startAllPresent = areTxnsInStartSample(startTxns);
+                startSample += 10;
+                if(startSample > 1000)
+                {
+                    startAllPresent = InIndex.NO;
+                    break;
+                }
+            }
+            while(startAllPresent == InIndex.INDETERMINATE);
+            
+            int endSample = 10;
+            InIndex endAllPresent;
+            do
+            {
+                List<Transaction> endTxns = nodeDaoService.getTxnsByCommitTimeDescending(
+                        Long.MIN_VALUE, Long.MAX_VALUE, endSample, null, false);
+                endAllPresent = areAllTxnsInEndSample(endTxns);
+                endSample += 10;
+                if(endSample > 1000)
+                {
+                    endAllPresent = InIndex.NO;
+                    break;
+                }
+            }
+            while(endAllPresent == InIndex.INDETERMINATE);
             
             // check the level of cover required
             switch (recoveryMode)
             {
             case AUTO:
-                if (!startAllPresent)
+                if (startAllPresent == InIndex.NO)
                 {
                     // Initial transactions are missing - rebuild
                     performFullRecovery();
                 }
-                else if (!endAllPresent)
+                else if (endAllPresent == InIndex.NO)
                 {
                     performPartialRecovery();
                 }
                 break;
             case VALIDATE:
                 // Check
-                if (!startAllPresent || !endAllPresent)
+                if ((startAllPresent == InIndex.NO) || (endAllPresent == InIndex.NO))
                 {
                     // Index is out of date
                     logger.warn(I18NUtil.getMessage(ERR_INDEX_OUT_OF_DATE));
@@ -225,6 +251,50 @@ public class FullIndexRecoveryComponent extends AbstractReindexComponent
         
     }
     
+    /**
+     * @return          Returns <tt>false</tt> if any one of the transactions aren't in the index.
+     */
+    protected InIndex areAllTxnsInEndSample(List<Transaction> txns)
+    {
+        int yesCount = 0;
+        for (Transaction txn : txns)
+        {
+            if (isTxnPresentInIndex(txn) == InIndex.NO)
+            {
+                // Missing txn
+                return InIndex.NO;
+            }
+            if (isTxnPresentInIndex(txn) == InIndex.YES)
+            {
+                yesCount++;
+            }
+        }
+        // Work around for TX that is written to the repo at start up .. must be more than one real add /update
+        if(yesCount > 1)
+        {
+            return InIndex.YES;
+        }
+        else
+        {
+            return InIndex.INDETERMINATE;
+        }
+    }
+    
+    protected InIndex areTxnsInStartSample(List<Transaction> txns)
+    {
+        InIndex current = InIndex.INDETERMINATE;
+        for (Transaction txn : txns)
+        {
+            current = isTxnPresentInIndex(txn);
+            if (current == InIndex.NO)
+            {
+                // Missing txn
+                return InIndex.NO;
+            } 
+        }
+        return current;
+    }
+    
     private void performPartialRecovery()
     {
         // Log the AUTO recovery

source/java/org/alfresco/repo/node/index/FullIndexRecoveryComponentTest.java

@@ -18,8 +18,27 @@
  */
 package org.alfresco.repo.node.index;
 
+import java.util.List;
+import java.util.Locale;
+
+import javax.transaction.UserTransaction;
+
 import junit.framework.TestCase;
 
+import org.alfresco.i18n.I18NUtil;
+import org.alfresco.model.ContentModel;
+import org.alfresco.repo.dictionary.NamespaceDAOImpl;
+import org.alfresco.repo.domain.Transaction;
+import org.alfresco.repo.node.db.NodeDaoService;
+import org.alfresco.repo.node.index.AbstractReindexComponent.InIndex;
+import org.alfresco.repo.security.authentication.AuthenticationComponent;
+import org.alfresco.repo.transaction.RetryingTransactionHelper;
+import org.alfresco.service.ServiceRegistry;
+import org.alfresco.service.cmr.repository.NodeRef;
+import org.alfresco.service.cmr.repository.NodeService;
+import org.alfresco.service.cmr.repository.StoreRef;
+import org.alfresco.service.namespace.QName;
+import org.alfresco.service.transaction.TransactionService;
 import org.alfresco.util.ApplicationContextHelper;
 import org.springframework.context.ApplicationContext;
 
@@ -35,10 +54,37 @@ public class FullIndexRecoveryComponentTest extends TestCase
     private FullIndexRecoveryComponent indexRecoverer;
 
     private AVMFullIndexRecoveryComponent avmIndexRecoveryComponent;
+
+    private NodeService nodeService;
+
+    private NodeRef rootNodeRef;
+
+    private TransactionService transactionService;
+
+    private RetryingTransactionHelper retryingTransactionHelper;
+
+    private AuthenticationComponent authenticationComponent;
+
+    private UserTransaction testTX;
+    
+    private NodeDaoService nodeDaoService;
+    
     public void setUp() throws Exception
     {
         indexRecoverer = (FullIndexRecoveryComponent) ctx.getBean("indexRecoveryComponent");
         avmIndexRecoveryComponent = (AVMFullIndexRecoveryComponent) ctx.getBean("avmIndexRecoveryComponent");
+        nodeService = (NodeService) ctx.getBean("nodeService");
+        transactionService = (TransactionService) ctx.getBean("transactionComponent");
+        retryingTransactionHelper = (RetryingTransactionHelper) ctx.getBean("retryingTransactionHelper");
+        authenticationComponent = (AuthenticationComponent) ctx.getBean("authenticationComponent");     
+        nodeDaoService = (NodeDaoService) ctx.getBean("nodeDaoServiceImpl"); 
+
+        testTX = transactionService.getUserTransaction();
+        testTX.begin();
+        this.authenticationComponent.setSystemUserAsCurrentUser();
+        
+        
+     
         
     }
     
@@ -47,6 +93,75 @@ public class FullIndexRecoveryComponentTest extends TestCase
         
     }
     
+    public void XtestDeletionReporting() throws Exception
+    {
+        StoreRef storeRef = nodeService.createStore(StoreRef.PROTOCOL_WORKSPACE, "Test_" + System.currentTimeMillis());
+        rootNodeRef = nodeService.getRootNode(storeRef);
+
+        NodeRef folder = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}folder"), ContentModel.TYPE_FOLDER).getChildRef();
+
+        testTX.commit();
+        testTX = transactionService.getUserTransaction();
+        testTX.begin();
+        
+        NodeRef[] refs = new NodeRef[20];
+        for(int i = 0; i < refs.length; i++)
+        {
+            refs[i] = nodeService.createNode(folder, ContentModel.ASSOC_CONTAINS, QName.createQName("{namespace}file"+i), ContentModel.TYPE_CONTENT).getChildRef();
+        }
+        
+        testTX.commit();
+        testTX = transactionService.getUserTransaction();
+        testTX.begin();
+        this.authenticationComponent.setSystemUserAsCurrentUser();
+        for(int i = 0; i < refs.length; i++)
+        {
+            nodeService.deleteNode(refs[i]);
+            testTX.commit();
+            testTX = transactionService.getUserTransaction();
+            testTX.begin();
+        }
+        
+        // The following test are important but take too long ....
+        
+//        List<Transaction> startTxns = nodeDaoService.getTxnsByCommitTimeAscending(
+//                Long.MIN_VALUE, Long.MAX_VALUE, 1, null, false);
+//        InIndex startAllPresent = indexRecoverer.areTxnsInStartSample(startTxns);
+//        assertEquals(InIndex.YES, startAllPresent);
+//        Long maxId = nodeDaoService.getMaxTxnIdByCommitTime(Long.MAX_VALUE);
+//        startTxns = nodeDaoService.getTxnsByCommitTimeAscending(
+//                Long.MIN_VALUE, Long.MAX_VALUE, maxId.intValue(), null, false);
+//        startAllPresent = indexRecoverer.areTxnsInStartSample(startTxns);
+//        assertEquals(InIndex.INDETERMINATE, startAllPresent);
+       
+//        for(int i = 0; i <= maxId.intValue(); i++)
+//        {
+//            System.out.println("TX "+i+" is "+indexRecoverer.isTxnPresentInIndex(nodeDaoService.getTxnById(i)));
+//        }
+        
+//        startTxns = nodeDaoService.getTxnsByCommitTimeAscending(
+//                Long.MIN_VALUE, Long.MAX_VALUE, startTxns.size() - 20, null, false);
+//        startAllPresent = indexRecoverer.areTxnsInStartSample(startTxns);
+//        assertEquals(InIndex.YES, startAllPresent);
+//        
+        
+        
+        List<Transaction> endTxns = nodeDaoService.getTxnsByCommitTimeDescending(
+                Long.MIN_VALUE, Long.MAX_VALUE, 20, null, false);
+        InIndex endAllPresent = indexRecoverer.areAllTxnsInEndSample(endTxns);
+        assertEquals(InIndex.INDETERMINATE, endAllPresent);
+        
+        endTxns = nodeDaoService.getTxnsByCommitTimeDescending(
+                Long.MIN_VALUE, Long.MAX_VALUE, 21, null, false);
+        endAllPresent = indexRecoverer.areAllTxnsInEndSample(endTxns);
+        assertEquals(InIndex.INDETERMINATE, endAllPresent);
+        
+        endTxns = nodeDaoService.getTxnsByCommitTimeDescending(
+                Long.MIN_VALUE, Long.MAX_VALUE, 22, null, false);
+        endAllPresent = indexRecoverer.areAllTxnsInEndSample(endTxns);
+        assertEquals(InIndex.YES, endAllPresent);
+    }
+    
     public synchronized void testReindexing() throws Exception
     {
         indexRecoverer.setRecoveryMode(FullIndexRecoveryComponent.RecoveryMode.FULL.name());

source/java/org/alfresco/repo/node/index/IndexTransactionTracker.java

@@ -456,11 +456,16 @@ found:
                 case YES:
                     fromTimeInclusive = txnCommitTime;
                     break found;
+                case INDETERMINATE:
+                    // If we hit an indeterminate transaction we go back a small amount to try and hit something definitive before a bigger step back
+                    firstWasInIndex = false;
+                    toTimeExclusive = txnCommitTime - 1000;
+                    continue;
                 default:
                     firstWasInIndex = false;
                     // Look further back in time.  Step back by 60 seconds each time, increasing
                     // the step by 10% each iteration.
-                    // Don't step back by more than a day
+                    // Don't step back by more than an hour
                     long decrement = Math.min(ONE_HOUR_MS, (long) (60000.0D * stepFactor));
                     toTimeExclusive = txnCommitTime - decrement;
                     stepFactor *= 1.1D;