Fix for ALF-786: WCM Cluster / Lucene: Searching in staging returns duplicate entries (when concurrently submtting)

- fixed duplication arising from indexing 0-1 and 0-2 against the index for near simultaneous snapshots 
- tracking has not been modified nor has index rebuild - the latest index info from the AVM index is definitive and complete - we do not have to do additional work as we do for DM

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@19898 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/repo/avm/AVMServiceTest.java

@@ -29,11 +29,15 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
 import java.util.TreeMap;
 
+import javax.transaction.UserTransaction;
+
 import org.alfresco.config.JNDIConstants;
 import org.alfresco.model.ContentModel;
 import org.alfresco.model.WCMModel;
@@ -47,6 +51,8 @@ import org.alfresco.repo.avm.actions.SimpleAVMSubmitAction;
 import org.alfresco.repo.avm.util.BulkLoader;
 import org.alfresco.repo.content.MimetypeMap;
 import org.alfresco.repo.domain.PropertyValue;
+import org.alfresco.repo.domain.hibernate.SessionSizeResourceManager;
+import org.alfresco.repo.search.impl.lucene.ADMLuceneSearcherImpl;
 import org.alfresco.repo.search.impl.lucene.LuceneQueryParser;
 import org.alfresco.repo.security.authentication.AuthenticationUtil;
 import org.alfresco.repo.transaction.RetryingTransactionHelper;
@@ -65,6 +71,7 @@ import org.alfresco.service.cmr.avmsync.AVMDifference;
 import org.alfresco.service.cmr.dictionary.DataTypeDefinition;
 import org.alfresco.service.cmr.model.FileFolderService;
 import org.alfresco.service.cmr.remote.RepoRemote;
+import org.alfresco.service.cmr.repository.ChildAssociationRef;
 import org.alfresco.service.cmr.repository.ContentData;
 import org.alfresco.service.cmr.repository.ContentWriter;
 import org.alfresco.service.cmr.repository.CrossRepositoryCopyService;
@@ -95,6 +102,430 @@ public class AVMServiceTest extends AVMServiceTestBase
         super.testSetup();
     }
     
+    public void test_ALF_786() throws Exception
+    {
+        int threads= 4;
+        int loops = 10;
+        int snapshotsPerLoop = 4;
+        
+        
+        int startVersion;
+        UserTransaction testTX = fTransactionService.getUserTransaction();
+        testTX.begin();
+        fService.createDirectory("main:/", "test");
+        startVersion = fService.createSnapshot("main", null, null).get("main");
+        
+        testTX.commit();
+        testTX = fTransactionService.getUserTransaction();
+        testTX.begin();
+
+        
+        StoreRef storeRef = AVMNodeConverter.ToStoreRef("main");
+        SearchService searchService = fIndexerAndSearcher.getSearcher(AVMNodeConverter.ToStoreRef("main"), true);
+        ResultSet results = searchService.query(storeRef, "lucene", "PATH:\"/test/*\"");
+        assertEquals(0, results.length());
+        results.close();
+        testTX.commit();
+
+        Thread runner = null;
+
+        for (int i = 0; i < threads; i++)
+        {
+            runner = new Nester("Concurrent-" + i, runner, false, snapshotsPerLoop, Nester.Mode.CREATE, loops);
+        }
+        if (runner != null)
+        {
+            runner.start();
+
+            try
+            {
+                runner.join();
+            }
+            catch (InterruptedException e)
+            {
+                e.printStackTrace();
+            }
+        }
+
+        testTX = fTransactionService.getUserTransaction();
+        testTX.begin();
+        // snap
+        testTX.commit();
+        testTX = fTransactionService.getUserTransaction();
+        testTX.begin();;
+        SortedMap<String, AVMNodeDescriptor> listing = fService.getDirectoryListing(-1, "main:/test");
+        assertEquals(loops, listing.size());
+        for(AVMNodeDescriptor node : listing.values())
+        {
+            System.out.println("Listed: "+node.getPath()+" "+node.getVersionID()); 
+        }
+        List<AVMDifference> diffs = fSyncService.compare(startVersion, "main:/", -1, "main:/", null);
+        assertEquals(loops, diffs.size());
+        for(AVMDifference diff : diffs)
+        {
+            AVMNodeDescriptor desc = fService.lookup(diff.getDestinationVersion(), diff.getDestinationPath(), true);
+            assertFalse(desc.isDeleted());
+        }
+        
+        
+        
+        searchService = fIndexerAndSearcher.getSearcher(AVMNodeConverter.ToStoreRef("main"), true);
+        results = searchService.query(storeRef, "lucene", "PATH:\"/test/*\"");
+        for(ResultSetRow row : results)
+        {
+            System.out.println("Found: "+row.getNodeRef());
+        }
+        assertEquals(loops, results.length());
+        results.close();
+        testTX.commit();
+        
+        // update
+        
+        runner = null;
+        for (int i = 0; i < threads; i++)
+        {
+            runner = new Nester("Concurrent-" + i, runner, false, snapshotsPerLoop, Nester.Mode.UPDATE, loops);
+        }
+        if (runner != null)
+        {
+            runner.start();
+
+            try
+            {
+                runner.join();
+            }
+            catch (InterruptedException e)
+            {
+                e.printStackTrace();
+            }
+        }
+        
+        testTX = fTransactionService.getUserTransaction();
+        testTX.begin();
+        searchService = fIndexerAndSearcher.getSearcher(AVMNodeConverter.ToStoreRef("main"), true);
+        results = searchService.query(storeRef, "lucene", "PATH:\"/test/*\"");
+        for(ResultSetRow row : results)
+        {
+            System.out.println("Found: "+row.getNodeRef());
+        }
+        assertEquals(loops, results.length());
+        results.close();
+        testTX.commit();
+        
+         // delete
+        
+        runner = null;
+        for (int i = 0; i < threads; i++)
+        {
+            runner = new Nester("Concurrent-" + i, runner, false, snapshotsPerLoop, Nester.Mode.DELETE, loops);
+        }
+        if (runner != null)
+        {
+            runner.start();
+
+            try
+            {
+                runner.join();
+            }
+            catch (InterruptedException e)
+            {
+                e.printStackTrace();
+            }
+        }
+        
+        testTX = fTransactionService.getUserTransaction();
+        testTX.begin();
+        searchService = fIndexerAndSearcher.getSearcher(AVMNodeConverter.ToStoreRef("main"), true);
+        results = searchService.query(storeRef, "lucene", "PATH:\"/test/*\"");
+        for(ResultSetRow row : results)
+        {
+            System.out.println("Found: "+row.getNodeRef());
+        }
+        assertEquals(0, results.length());
+        results.close();
+        testTX.commit();
+        
+        // recreate
+        
+        runner = null;
+        for (int i = 0; i < threads; i++)
+        {
+            runner = new Nester("Concurrent-" + i, runner, false, snapshotsPerLoop, Nester.Mode.CREATE, loops);
+        }
+        if (runner != null)
+        {
+            runner.start();
+
+            try
+            {
+                runner.join();
+            }
+            catch (InterruptedException e)
+            {
+                e.printStackTrace();
+            }
+        }
+        
+        testTX = fTransactionService.getUserTransaction();
+        testTX.begin();
+        searchService = fIndexerAndSearcher.getSearcher(AVMNodeConverter.ToStoreRef("main"), true);
+        results = searchService.query(storeRef, "lucene", "PATH:\"/test/*\"");
+        for(ResultSetRow row : results)
+        {
+            System.out.println("Found: "+row.getNodeRef());
+        }
+        assertEquals(loops, results.length());
+        results.close();
+        testTX.commit();
+        
+        //move
+        
+        runner = null;
+        for (int i = 0; i < threads; i++)
+        {
+            runner = new Nester("Concurrent-" + i, runner, false, snapshotsPerLoop, Nester.Mode.MOVE, loops);
+        }
+        if (runner != null)
+        {
+            runner.start();
+
+            try
+            {
+                runner.join();
+            }
+            catch (InterruptedException e)
+            {
+                e.printStackTrace();
+            }
+        }
+        
+        testTX = fTransactionService.getUserTransaction();
+        testTX.begin();
+        searchService = fIndexerAndSearcher.getSearcher(AVMNodeConverter.ToStoreRef("main"), true);
+        results = searchService.query(storeRef, "lucene", "PATH:\"/test/*\"");
+        for(ResultSetRow row : results)
+        {
+            System.out.println("Found: "+row.getNodeRef());
+        }
+        assertEquals(loops, results.length());
+        results.close();
+        testTX.commit();
+    }
+    
+    public void xtest_ALF_786_PLUS() throws Exception
+    {
+        int startVersion;
+        UserTransaction testTX = fTransactionService.getUserTransaction();
+        testTX.begin();
+        fService.createDirectory("main:/", "test");
+        startVersion = fService.createSnapshot("main", null, null).get("main");
+        
+        testTX.commit();
+        testTX = fTransactionService.getUserTransaction();
+        testTX.begin();
+
+        
+        StoreRef storeRef = AVMNodeConverter.ToStoreRef("main");
+        SearchService searchService = fIndexerAndSearcher.getSearcher(AVMNodeConverter.ToStoreRef("main"), true);
+        ResultSet results = searchService.query(storeRef, "lucene", "PATH:\"/test/*\"");
+        assertEquals(0, results.length());
+        results.close();
+        testTX.commit();
+
+        Thread runner = null;
+
+        for (int i = 0; i < 10; i++)
+        {
+            runner = new Nester("Concurrent-" + i, runner, true, 10, Nester.Mode.CREATE, 10 );
+        }
+        if (runner != null)
+        {
+            runner.start();
+
+            try
+            {
+                runner.join();
+            }
+            catch (InterruptedException e)
+            {
+                e.printStackTrace();
+            }
+        }
+
+        testTX = fTransactionService.getUserTransaction();
+        testTX.begin();
+        // snap
+        testTX.commit();
+        testTX = fTransactionService.getUserTransaction();
+        testTX.begin();;
+        SortedMap<String, AVMNodeDescriptor> listing = fService.getDirectoryListing(-1, "main:/test");
+        assertEquals(100, listing.size());
+        for(AVMNodeDescriptor node : listing.values())
+        {
+            System.out.println("Listed: "+node.getPath()+" "+node.getVersionID()); 
+        }
+        List<AVMDifference> diffs = fSyncService.compare(startVersion, "main:/", -1, "main:/", null);
+        assertEquals(100, diffs.size());
+        for(AVMDifference diff : diffs)
+        {
+            AVMNodeDescriptor desc = fService.lookup(diff.getDestinationVersion(), diff.getDestinationPath(), true);
+            assertFalse(desc.isDeleted());
+        }
+        
+        
+        
+        searchService = fIndexerAndSearcher.getSearcher(AVMNodeConverter.ToStoreRef("main"), true);
+        results = searchService.query(storeRef, "lucene", "PATH:\"/test/*\"");
+        for(ResultSetRow row : results)
+        {
+            System.out.println("Found: "+row.getNodeRef());
+        }
+        assertEquals(100, results.length());
+        results.close();
+        testTX.commit();
+    }
+    
+    static class Nester extends Thread
+    {
+        enum Mode {CREATE, UPDATE, DELETE, MOVE};
+        
+        Thread waiter;
+
+        int i;
+        
+        boolean multiThread;
+        
+        int snapshotCount;
+        
+        Mode mode;
+        
+        int loopCount;
+        
+        Nester(String name, Thread waiter, boolean multiThread, int snapshotCount, Mode mode, int loopCount)
+        {
+            super(name);
+            this.setDaemon(true);
+            this.waiter = waiter;
+            this.multiThread = multiThread;
+            this.snapshotCount = snapshotCount;
+            this.mode = mode;
+            this.loopCount = loopCount;
+        }
+
+        public void run()
+        {
+            fAuthenticationComponent.setSystemUserAsCurrentUser();
+            if (waiter != null)
+            {
+                waiter.start();
+            }
+            try
+            {
+                System.out.println("Start " + this.getName());
+                for(i = 0; i < loopCount; i++)
+                {
+                    RetryingTransactionCallback<Void> create = new RetryingTransactionCallback<Void>()
+                    {
+                        public Void execute() throws Throwable
+                        {
+                            fService.createFile("main:/test", getName()+"-"+i);
+
+                            return null;
+                        }
+                    };
+                    RetryingTransactionCallback<Void> update = new RetryingTransactionCallback<Void>()
+                    {
+                        public Void execute() throws Throwable
+                        {
+                            fService.setMimeType("main:/test/"+getName()+"-"+i, "text/plain");
+
+                            return null;
+                        }
+                    };
+                    RetryingTransactionCallback<Void> delete = new RetryingTransactionCallback<Void>()
+                    {
+                        public Void execute() throws Throwable
+                        {
+                            fService.removeNode("main:/test/"+getName()+"-"+i);
+
+                            return null;
+                        }
+                    };
+                    RetryingTransactionCallback<Void> move = new RetryingTransactionCallback<Void>()
+                    {
+                        public Void execute() throws Throwable
+                        {
+                            fService.rename("main:/test/", getName()+"-"+i, "main:/test/", "MOVED-"+getName()+"-"+i);
+
+                            return null;
+                        }
+                    };
+                    if(multiThread || (waiter == null))
+                    {
+                         // only one thread creates for 786
+                        switch(mode)
+                        {
+                        case CREATE:
+                            fRetryingTransactionHelper.doInTransaction(create);
+                            System.out.println(getName()+i);
+                            break;
+                        case UPDATE:
+                            fRetryingTransactionHelper.doInTransaction(update);
+                            break;
+                        case DELETE:
+                            fRetryingTransactionHelper.doInTransaction(delete);
+                            break;
+                        case MOVE:
+                            fRetryingTransactionHelper.doInTransaction(move);
+                            break;
+                        default:
+                        }
+                       
+                    }
+                   
+                    RetryingTransactionCallback<Void> snap = new RetryingTransactionCallback<Void>()
+                    {
+                        public Void execute() throws Throwable
+                        {
+                            fService.createSnapshot("main", null, null);
+                           
+
+                            return null;
+                        }
+                    };
+                    for(int s = 0; s < snapshotCount; s++)
+                    {
+                        fRetryingTransactionHelper.doInTransaction(snap);
+                    }
+                }
+                System.out.println("End " + this.getName());
+            }
+            catch (Exception e)
+            {
+                System.out.println("End " + this.getName() + " with error " + e.getMessage());
+                e.printStackTrace();
+            }
+            finally
+            {
+                fAuthenticationComponent.clearCurrentSecurityContext();
+            }
+            if (waiter != null)
+            {
+                try
+                {
+                    waiter.join();
+                    System.out.println("Waited for " + waiter.getName()+" by "+this.getName());
+                }
+                catch (InterruptedException e)
+                {
+                }
+            }
+        }
+
+    }
+    
+    
+    
     public void testDiffOrder()
     {
         try
@@ -348,6 +779,57 @@ public class AVMServiceTest extends AVMServiceTestBase
             assertEquals(1, fService.getStoreVersions("main", null, new Date(times.get(0))).size());
             assertEquals(3, fService.getStoreVersions("main", new Date(times.get(0)), null).size());
             assertEquals(2, fService.getStoreVersions("main", new Date(times.get(1)), new Date(System.currentTimeMillis())).size());
+            
+            // TO
+            assertEquals(1, fService.getStoreVersionsTo("main", 0).size());
+            assertEquals(0, fService.getStoreVersionsTo("main", 0).get(0).getVersionID());
+            assertEquals(2, fService.getStoreVersionsTo("main", 1).size());
+            assertEquals(0, fService.getStoreVersionsTo("main", 1).get(0).getVersionID());
+            assertEquals(1, fService.getStoreVersionsTo("main", 1).get(1).getVersionID());
+            assertEquals(3, fService.getStoreVersionsTo("main", 2).size());
+            assertEquals(0, fService.getStoreVersionsTo("main", 2).get(0).getVersionID());
+            assertEquals(1, fService.getStoreVersionsTo("main", 2).get(1).getVersionID());
+            assertEquals(2, fService.getStoreVersionsTo("main", 2).get(2).getVersionID());
+            assertEquals(4, fService.getStoreVersionsTo("main", 3).size());
+            assertEquals(0, fService.getStoreVersionsTo("main", 3).get(0).getVersionID());
+            assertEquals(1, fService.getStoreVersionsTo("main", 3).get(1).getVersionID());
+            assertEquals(2, fService.getStoreVersionsTo("main", 3).get(2).getVersionID());
+            assertEquals(3, fService.getStoreVersionsTo("main", 3).get(3).getVersionID());
+            
+            // FROM
+            
+            assertEquals(4, fService.getStoreVersionsFrom("main", 0).size());
+            assertEquals(0, fService.getStoreVersionsFrom("main", 0).get(0).getVersionID());
+            assertEquals(1, fService.getStoreVersionsFrom("main", 0).get(1).getVersionID());
+            assertEquals(2, fService.getStoreVersionsFrom("main", 0).get(2).getVersionID());
+            assertEquals(3, fService.getStoreVersionsFrom("main", 0).get(3).getVersionID());
+            assertEquals(3, fService.getStoreVersionsFrom("main", 1).size());
+            assertEquals(1, fService.getStoreVersionsFrom("main", 1).get(0).getVersionID());
+            assertEquals(2, fService.getStoreVersionsFrom("main", 1).get(1).getVersionID());
+            assertEquals(3, fService.getStoreVersionsFrom("main", 1).get(2).getVersionID());
+            assertEquals(2, fService.getStoreVersionsFrom("main", 2).size());
+            assertEquals(2, fService.getStoreVersionsFrom("main", 2).get(0).getVersionID());
+            assertEquals(3, fService.getStoreVersionsFrom("main", 2).get(1).getVersionID());
+            assertEquals(1, fService.getStoreVersionsFrom("main", 3).size());
+            assertEquals(3, fService.getStoreVersionsFrom("main", 3).get(0).getVersionID());
+            assertEquals(0, fService.getStoreVersionsFrom("main", 4).size());
+            
+            // BETWEEN
+            
+            assertEquals(1, fService.getStoreVersionsBetween("main", 0, 0).size());
+            assertEquals(0, fService.getStoreVersionsBetween("main", 0, 0).get(0).getVersionID());
+            assertEquals(1, fService.getStoreVersionsBetween("main", 1, 1).size());
+            assertEquals(1, fService.getStoreVersionsBetween("main", 1, 1).get(0).getVersionID());
+            assertEquals(1, fService.getStoreVersionsBetween("main", 2, 2).size());
+            assertEquals(2, fService.getStoreVersionsBetween("main", 2, 2).get(0).getVersionID());
+            assertEquals(1, fService.getStoreVersionsBetween("main", 3, 3).size());
+            assertEquals(3, fService.getStoreVersionsBetween("main", 3, 3).get(0).getVersionID());
+            assertEquals(0, fService.getStoreVersionsBetween("main", 4, 4).size());
+            
+            
+            assertEquals(2, fService.getStoreVersionsBetween("main", 1, 2).size());
+            assertEquals(1, fService.getStoreVersionsBetween("main", 1, 2).get(0).getVersionID());
+            assertEquals(2, fService.getStoreVersionsBetween("main", 1, 2).get(1).getVersionID());
         }
         catch (Exception e)
         {
@@ -2182,7 +2664,7 @@ public class AVMServiceTest extends AVMServiceTestBase
         {
             setupBasicTree();
             fService.removeNode("main:/a/b/c/foo/");
-            fService.removeNode("main://d");
+            fService.removeNode("main:/d");
             fService.createSnapshot("main", null, null);
 
             StoreRef storeRef = AVMNodeConverter.ToStoreRef("main");
@@ -2208,7 +2690,7 @@ public class AVMServiceTest extends AVMServiceTestBase
             // TODO: Fix auth in AVMDiskDriver and more??
 
             results = searchService.query(storeRef, "lucene", LuceneQueryParser.escape("@" + ContentModel.PROP_CREATOR) + ":admin");
-            if (results.length() == 6)
+            if (results.length() == 7)
             {
                 for (ResultSetRow row : results)
                 {