Index update check point

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@3070 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java

@@ -26,36 +26,42 @@ class IndexEntry
     /**
      * The type of the index entry
      */
-    IndexType type;
+    private IndexType type;
 
     /**
      * The unique name of the index entry
      */
-    String name;
+    private String name;
     
     /**
      * The preceeding index name.
      * Allows deltas etc to apply to the index or an overlay for example. 
      */
-    String parentName;
+    private String parentName;
 
     /**
-     * The status of the inedx entry
+     * The status of the index entry
      */
-    TransactionStatus status;
+    private TransactionStatus status;
     
     /**
      * If merging, the id where the result is going
      */
-    String mergeId;
+    private String mergeId;
-
+    
-    IndexEntry(IndexType type, String name,  String parentName, TransactionStatus status, String mergeId)
+    private long documentCount; 
+    
+    private long deletions;
+    
+    IndexEntry(IndexType type, String name,  String parentName, TransactionStatus status, String mergeId, long documentCount, long deletions)
     {
         this.type = type;
         this.name = name;
         this.parentName = parentName;
         this.status = status;
         this.mergeId = mergeId;
+        this.documentCount = documentCount;
+        this.deletions = deletions;
     }
 
     public String getMergeId()
@@ -107,6 +113,36 @@ class IndexEntry
     {
         this.type = type;
     }
+
+    public long getDocumentCount()
+    {
+        return documentCount;
+    }
+
+    public void setDocumentCount(long documentCount)
+    {
+        this.documentCount = documentCount;
+    }
+    
+    public long getDeletions()
+    {
+        return deletions;
+    }
+
+    public void setDeletions(long deletions)
+    {
+        this.deletions = deletions;
+    }
+
+    public String toString()
+    {
+        StringBuilder builder = new StringBuilder();
+        builder.append(" Name=").append(getName()).append(" ");
+        builder.append("Type=").append(getType()).append(" ");
+        builder.append("Status=").append(getStatus()).append(" ");
+        builder.append("Docs=").append(getDocumentCount()).append(" ");
+        return builder.toString();
+    }
     
     
 }

source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfoTest.java

@@ -0,0 +1,871 @@
+/*
+ * Copyright (C) 2005 Alfresco, Inc.
+ *
+ * Licensed under the Mozilla Public License version 1.1 
+ * with a permitted attribution clause. You may obtain a
+ * copy of the License at
+ *
+ *   http://www.alfresco.org/legal/license.txt
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific
+ * language governing permissions and limitations under the
+ * License.
+ */
+package org.alfresco.repo.search.impl.lucene.index;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+
+import org.alfresco.service.cmr.repository.NodeRef;
+import org.alfresco.service.cmr.repository.StoreRef;
+import org.alfresco.util.GUID;
+import org.alfresco.util.TempFileProvider;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+
+import junit.framework.TestCase;
+
+public class IndexInfoTest extends TestCase
+{
+
+    public static final String[] WORD_LIST = { "aardvark", "banana", "cucumber", "daffodil", "emu", "frog", "gibbon",
+            "humour", "injection", "jelly", "key", "lemur", "monkey", "number", "open", "plummet", "quest",
+            "replication", "steam", "tunnel", "uncommon", "verbose", "where", "xylem", "yellow", "zebra", "alpha",
+            "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "indigo", "juliet", "kilo", "lima",
+            "mike", "november", "oscar", "papa", "quebec", "romeo", "sierra", "tango", "uniform", "victor", "whisky",
+            "xray", "yankee", "zulu" };
+
+    public static final String[] CREATE_LIST = { "aardvark", "banana", "cucumber", "daffodil", "emu", "frog", "gibbon",
+            "humour", "injection", "jelly", "key", "lemur", "monkey", "number", "open", "plummet", "quest",
+            "replication", "steam", "tunnel", "uncommon", "verbose", "where", "xylem", "yellow", "zebra", };
+
+    public static final String[] UPDATE_LIST = { "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf",
+            "hotel", "indigo", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa", "quebec", "romeo",
+            "sierra", "tango", "uniform", "victor", "whisky", "xray", "yankee", "zulu" };
+    
+    public static final String[] CREATE_LIST_2 = { "aardvark2", "banana2", "cucumber2", "daffodil2", "emu2", "frog2", "gibbon2",
+        "humour2", "injection2", "jelly2", "key2", "lemur2", "monkey2", "number2", "open2", "plummet2", "quest2",
+        "replication2", "steam2", "tunnel2", "uncommon2", "verbose2", "where2", "xylem2", "yellow2", "zebra2", };
+
+public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", "delta2", "echo2", "foxtrot2", "golf2",
+        "hotel2", "indigo2", "juliet2", "kilo2", "lima2", "mike2", "november2", "oscar2", "papa2", "quebec2", "romeo2",
+        "sierra2", "tango2", "uniform2", "victor2", "whisky2", "xray2", "yankee2", "zulu2" };
+
+    public IndexInfoTest()
+    {
+        super();
+    }
+
+    public IndexInfoTest(String arg0)
+    {
+        super(arg0);
+    }
+
+    public void testCreateAndSearch() throws IOException
+    {
+        System.setProperty("disableLuceneLocks", "true");
+
+        // no deletions - create only
+        HashSet<NodeRef> deletions = new HashSet<NodeRef>();
+        for (int i = 0; i < 0; i++)
+        {
+            deletions.add(new NodeRef(new StoreRef("woof", "bingle"), GUID.generate()));
+        }
+
+        File tempLocation = TempFileProvider.getTempDir();
+        File testArea = new File(tempLocation, "IndexInfoTest");
+        File testDir = new File(testArea, "" + System.currentTimeMillis());
+        final IndexInfo ii = new IndexInfo(testDir);
+
+        for (int i = 0; i < WORD_LIST.length; i++)
+        {
+            IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), i);
+            reader.close();
+
+            String guid = GUID.generate();
+            ii.setStatus(guid, TransactionStatus.ACTIVE, null, null);
+            IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer());
+
+            Document doc = new Document();
+            for (int k = 0; k < 15; k++)
+            {
+                doc.add(new Field("ID" + k, guid, false, true, false));
+            }
+            doc.add(new Field("TEXT", WORD_LIST[i], false, true, false));
+            writer.addDocument(doc);
+
+            ii.closeDeltaIndexWriter(guid);
+            ii.setStatus(guid, TransactionStatus.PREPARING, null, null);
+            ii.setPreparedState(guid, deletions, 1);
+            ii.getDeletions(guid);
+            ii.setStatus(guid, TransactionStatus.PREPARED, null, null);
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), i);
+            for (int j = 0; j < WORD_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", WORD_LIST[j]));
+                if (j < i)
+                {
+                    assertTrue(tds.next());
+                    assertEquals(tds.doc(), j);
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions);
+            assertEquals(reader.numDocs(), i + 1);
+            for (int j = 0; j < WORD_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", WORD_LIST[j]));
+                if (j <= i)
+                {
+                    assertTrue(tds.next());
+                    assertEquals(tds.doc(), j);
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+            ii.setStatus(guid, TransactionStatus.COMMITTING, null, null);
+            ii.setStatus(guid, TransactionStatus.COMMITTED, null, null);
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), i + 1);
+            for (int j = 0; j < WORD_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", WORD_LIST[j]));
+                if (j <= i)
+                {
+                    assertTrue(tds.next());
+                    assertEquals(tds.doc(), j);
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+        }
+
+    }
+
+    public void testCreateDeleteAndSearch() throws IOException
+    {
+        assertEquals(CREATE_LIST.length, UPDATE_LIST.length);
+
+        StoreRef storeRef = new StoreRef("woof", "bingle");
+
+        System.setProperty("disableLuceneLocks", "true");
+
+        // no deletions - create only
+        ArrayList<NodeRef> nodeRefs = new ArrayList<NodeRef>();
+
+        File tempLocation = TempFileProvider.getTempDir();
+        File testArea = new File(tempLocation, "IndexInfoTest");
+        File testDir = new File(testArea, "" + System.currentTimeMillis());
+        final IndexInfo ii = new IndexInfo(testDir);
+
+        for (int i = 0; i < CREATE_LIST.length; i++)
+        {
+            IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), i);
+            reader.close();
+
+            String guid = GUID.generate();
+            ii.setStatus(guid, TransactionStatus.ACTIVE, null, null);
+            IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer());
+
+            Document doc = new Document();
+            for (int k = 0; k < 15; k++)
+            {
+                doc.add(new Field("ID" + k, guid, false, true, false));
+            }
+            doc.add(new Field("TEXT", CREATE_LIST[i], false, true, false));
+            NodeRef nodeRef = new NodeRef(storeRef, GUID.generate());
+            nodeRefs.add(nodeRef);
+            doc.add(new Field("ID", nodeRef.toString(), false, true, false));
+            writer.addDocument(doc);
+
+            ii.closeDeltaIndexWriter(guid);
+            ii.setStatus(guid, TransactionStatus.PREPARING, null, null);
+            ii.setPreparedState(guid, new HashSet<NodeRef>(), 1);
+            ii.getDeletions(guid);
+            ii.setStatus(guid, TransactionStatus.PREPARED, null, null);
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), i);
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j < i)
+                {
+                    assertTrue(tds.next());
+                    assertEquals(tds.doc(), j);
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet<NodeRef>());
+            assertEquals(reader.numDocs(), i + 1);
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j <= i)
+                {
+                    assertTrue(tds.next());
+                    assertEquals(tds.doc(), j);
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+            ii.setStatus(guid, TransactionStatus.COMMITTING, null, null);
+            ii.setStatus(guid, TransactionStatus.COMMITTED, null, null);
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), i + 1);
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j <= i)
+                {
+                    assertTrue(tds.next());
+                    assertEquals(tds.doc(), j);
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+        }
+
+        for (int i = 0; i < CREATE_LIST.length; i++)
+        {
+            HashSet<NodeRef> deletions = new HashSet<NodeRef>();
+            deletions.add(nodeRefs.get(i));
+
+            IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), CREATE_LIST.length - i);
+            reader.close();
+
+            String guid = GUID.generate();
+            ii.setStatus(guid, TransactionStatus.ACTIVE, null, null);
+            ii.closeDeltaIndexWriter(guid);
+            ii.setStatus(guid, TransactionStatus.PREPARING, null, null);
+            ii.setPreparedState(guid, deletions, 1);
+            ii.getDeletions(guid);
+            ii.setStatus(guid, TransactionStatus.PREPARED, null, null);
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), CREATE_LIST.length - i);
+            int lastDoc = -1;
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j >= i)
+                {
+                    assertTrue(tds.next());
+                    assertTrue(tds.doc() > lastDoc);
+                    lastDoc = tds.doc();
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions);
+            assertEquals(reader.numDocs(), UPDATE_LIST.length - i - 1);
+            lastDoc = -1;
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j > i)
+                {
+                    assertTrue(tds.next());
+                    assertTrue(tds.doc() > lastDoc);
+                    lastDoc = tds.doc();
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+
+            reader.close();
+
+            ii.setStatus(guid, TransactionStatus.COMMITTING, null, null);
+            ii.setStatus(guid, TransactionStatus.COMMITTED, null, null);
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), UPDATE_LIST.length - i - 1);
+            lastDoc = -1;
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j > i)
+                {
+                    assertTrue(tds.next());
+                    assertTrue(tds.doc() > lastDoc);
+                    lastDoc = tds.doc();
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+
+            reader.close();
+
+            IndexReader reader1 = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            IndexReader reader2 = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            IndexReader reader3 = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            reader3.close();
+            reader2.close();
+            reader1.close();
+
+        }
+
+    }
+
+    public void testCreateUpdateAndSearch() throws IOException
+    {
+        assertEquals(CREATE_LIST.length, UPDATE_LIST.length);
+
+        StoreRef storeRef = new StoreRef("woof", "bingle");
+
+        System.setProperty("disableLuceneLocks", "true");
+
+        // no deletions - create only
+        ArrayList<NodeRef> nodeRefs = new ArrayList<NodeRef>();
+
+        File tempLocation = TempFileProvider.getTempDir();
+        File testArea = new File(tempLocation, "IndexInfoTest");
+        File testDir = new File(testArea, "" + System.currentTimeMillis());
+        final IndexInfo ii = new IndexInfo(testDir);
+
+        for (int i = 0; i < CREATE_LIST.length; i++)
+        {
+            IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), i);
+            reader.close();
+
+            String guid = GUID.generate();
+            ii.setStatus(guid, TransactionStatus.ACTIVE, null, null);
+            IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer());
+
+            Document doc = new Document();
+            for (int k = 0; k < 15; k++)
+            {
+                doc.add(new Field("ID" + k, guid, false, true, false));
+            }
+            doc.add(new Field("TEXT", CREATE_LIST[i], false, true, false));
+            NodeRef nodeRef = new NodeRef(storeRef, GUID.generate());
+            nodeRefs.add(nodeRef);
+            doc.add(new Field("ID", nodeRef.toString(), false, true, false));
+            writer.addDocument(doc);
+
+            ii.closeDeltaIndexWriter(guid);
+            ii.setStatus(guid, TransactionStatus.PREPARING, null, null);
+            ii.setPreparedState(guid, new HashSet<NodeRef>(), 1);
+            ii.getDeletions(guid);
+            ii.setStatus(guid, TransactionStatus.PREPARED, null, null);
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), i);
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j < i)
+                {
+                    assertTrue(tds.next());
+                    assertEquals(tds.doc(), j);
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet<NodeRef>());
+            assertEquals(reader.numDocs(), i + 1);
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j <= i)
+                {
+                    assertTrue(tds.next());
+                    assertEquals(tds.doc(), j);
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+            ii.setStatus(guid, TransactionStatus.COMMITTING, null, null);
+            ii.setStatus(guid, TransactionStatus.COMMITTED, null, null);
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), i + 1);
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j <= i)
+                {
+                    assertTrue(tds.next());
+                    assertEquals(tds.doc(), j);
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+        }
+
+        for (int i = 0; i < UPDATE_LIST.length; i++)
+        {
+            HashSet<NodeRef> deletions = new HashSet<NodeRef>();
+            deletions.add(nodeRefs.get(i));
+
+            IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), UPDATE_LIST.length);
+            reader.close();
+
+            String guid = GUID.generate();
+            ii.setStatus(guid, TransactionStatus.ACTIVE, null, null);
+            IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer());
+
+            Document doc = new Document();
+            for (int k = 0; k < 15; k++)
+            {
+                doc.add(new Field("ID" + k, guid, false, true, false));
+            }
+            doc.add(new Field("TEXT", UPDATE_LIST[i], false, true, false));
+            writer.addDocument(doc);
+
+            ii.closeDeltaIndexWriter(guid);
+            ii.setStatus(guid, TransactionStatus.PREPARING, null, null);
+            ii.setPreparedState(guid, deletions, 1);
+            ii.getDeletions(guid);
+            ii.setStatus(guid, TransactionStatus.PREPARED, null, null);
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), UPDATE_LIST.length);
+            int lastDoc = -1;
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j >= i)
+                {
+                    assertTrue(tds.next());
+                    assertTrue(tds.doc() > lastDoc);
+                    lastDoc = tds.doc();
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            for (int j = 0; j < UPDATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", UPDATE_LIST[j]));
+                if (j < i)
+                {
+                    assertTrue(tds.next());
+                    assertTrue(tds.doc() > lastDoc);
+                    lastDoc = tds.doc();
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions);
+            assertEquals(reader.numDocs(), UPDATE_LIST.length);
+            lastDoc = -1;
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j > i)
+                {
+                    assertTrue(tds.next());
+                    assertTrue(tds.doc() > lastDoc);
+                    lastDoc = tds.doc();
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            for (int j = 0; j < UPDATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", UPDATE_LIST[j]));
+                if (j <= i)
+                {
+                    assertTrue(tds.next());
+                    assertTrue(tds.doc() > lastDoc);
+                    lastDoc = tds.doc();
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+
+            reader.close();
+
+            ii.setStatus(guid, TransactionStatus.COMMITTING, null, null);
+            ii.setStatus(guid, TransactionStatus.COMMITTED, null, null);
+
+            reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+            assertEquals(reader.numDocs(), UPDATE_LIST.length);
+            lastDoc = -1;
+            for (int j = 0; j < CREATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j]));
+                if (j > i)
+                {
+                    assertTrue(tds.next());
+                    assertTrue(tds.doc() > lastDoc);
+                    lastDoc = tds.doc();
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            for (int j = 0; j < UPDATE_LIST.length; j++)
+            {
+                TermDocs tds = reader.termDocs(new Term("TEXT", UPDATE_LIST[j]));
+                if (j <= i)
+                {
+                    assertTrue(tds.next());
+                    assertTrue(tds.doc() > lastDoc);
+                    lastDoc = tds.doc();
+                }
+                else
+                {
+                    assertFalse(tds.next());
+                }
+            }
+            reader.close();
+
+        }
+
+    }
+
+    public void testMultiThreadedCreateAndSearch()
+    {
+        
+        System.setProperty("disableLuceneLocks", "true");
+
+        File tempLocation = TempFileProvider.getTempDir();
+        File testArea = new File(tempLocation, "IndexInfoTest");
+        File testDir = new File(testArea, "" + System.currentTimeMillis());
+        final IndexInfo ii = new IndexInfo(testDir);
+        
+        Thread thread1 = new Thread(new Test(ii, CREATE_LIST, UPDATE_LIST));
+        Thread thread2 = new Thread(new Test(ii, CREATE_LIST_2, UPDATE_LIST_2));
+        thread1.start();
+        thread2.start();
+        try
+        {
+            thread1.join();
+            thread2.join();
+        }
+        catch (InterruptedException e)
+        {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    public static class Test implements Runnable
+    {
+        String[] create;
+        String[] update;
+        IndexInfo ii;
+        
+        Test(IndexInfo ii, String[] create, String[] update)
+        {
+            this.ii = ii;
+            this.create = create;
+            this.update = update;
+        }
+        
+        public void run()
+        {
+            try
+            {
+                assertEquals(create.length, update.length);
+                
+                StoreRef storeRef = new StoreRef("woof", "bingle");
+
+                // no deletions - create only
+                ArrayList<NodeRef> nodeRefs = new ArrayList<NodeRef>();
+
+                for (int i = 0; i < create.length; i++)
+                {
+                    IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+                    reader.close();
+
+                    String guid = GUID.generate();
+                    ii.setStatus(guid, TransactionStatus.ACTIVE, null, null);
+                    IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer());
+
+                    Document doc = new Document();
+                    for (int k = 0; k < 15; k++)
+                    {
+                        doc.add(new Field("ID" + k, guid, false, true, false));
+                    }
+                    doc.add(new Field("TEXT", create[i], false, true, false));
+                    NodeRef nodeRef = new NodeRef(storeRef, GUID.generate());
+                    nodeRefs.add(nodeRef);
+                    doc.add(new Field("ID", nodeRef.toString(), false, true, false));
+                    writer.addDocument(doc);
+
+                    ii.closeDeltaIndexWriter(guid);
+                    ii.setStatus(guid, TransactionStatus.PREPARING, null, null);
+                    ii.setPreparedState(guid, new HashSet<NodeRef>(), 1);
+                    ii.getDeletions(guid);
+                    ii.setStatus(guid, TransactionStatus.PREPARED, null, null);
+
+                    reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+                    
+                    int lastDoc = -1;
+                    
+                    for (int j = 0; j < create.length; j++)
+                    {
+                        TermDocs tds = reader.termDocs(new Term("TEXT", create[j]));
+                        if (j < i)
+                        {
+                            assertTrue(tds.next());
+                            assertTrue(tds.doc() > lastDoc);
+                            lastDoc = tds.doc();
+                        }
+                        else
+                        {
+                            assertFalse(tds.next());
+                        }
+                    }
+                    reader.close();
+
+                    reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet<NodeRef>());
+                    lastDoc = -1;
+                    for (int j = 0; j < create.length; j++)
+                    {
+                        TermDocs tds = reader.termDocs(new Term("TEXT", create[j]));
+                        if (j <= i)
+                        {
+                            assertTrue(tds.next());
+                            assertTrue(tds.doc() > lastDoc);
+                            lastDoc = tds.doc();
+                        }
+                        else
+                        {
+                            assertFalse(tds.next());
+                        }
+                    }
+                    reader.close();
+
+                    ii.setStatus(guid, TransactionStatus.COMMITTING, null, null);
+                    ii.setStatus(guid, TransactionStatus.COMMITTED, null, null);
+
+                    reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+                    lastDoc = -1;
+                    for (int j = 0; j < create.length; j++)
+                    {
+                        TermDocs tds = reader.termDocs(new Term("TEXT", create[j]));
+                        if (j <= i)
+                        {
+                            assertTrue(tds.next());
+                            assertTrue(tds.doc() > lastDoc);
+                            lastDoc = tds.doc();
+                        }
+                        else
+                        {
+                            assertFalse(tds.next());
+                        }
+                    }
+                    reader.close();
+
+                }
+
+                for (int i = 0; i < update.length; i++)
+                {
+                    HashSet<NodeRef> deletions = new HashSet<NodeRef>();
+                    deletions.add(nodeRefs.get(i));
+
+                    IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+                  
+                    reader.close();
+
+                    String guid = GUID.generate();
+                    ii.setStatus(guid, TransactionStatus.ACTIVE, null, null);
+                    IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer());
+
+                    Document doc = new Document();
+                    for (int k = 0; k < 15; k++)
+                    {
+                        doc.add(new Field("ID" + k, guid, false, true, false));
+                    }
+                    doc.add(new Field("TEXT", update[i], false, true, false));
+                    writer.addDocument(doc);
+
+                    ii.closeDeltaIndexWriter(guid);
+                    ii.setStatus(guid, TransactionStatus.PREPARING, null, null);
+                    ii.setPreparedState(guid, deletions, 1);
+                    ii.getDeletions(guid);
+                    ii.setStatus(guid, TransactionStatus.PREPARED, null, null);
+
+                    reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+                   
+                    int lastDoc = -1;
+                    for (int j = 0; j < create.length; j++)
+                    {
+                        TermDocs tds = reader.termDocs(new Term("TEXT", create[j]));
+                        if (j >= i)
+                        {
+                            assertTrue(tds.next());
+                            assertTrue(tds.doc() > lastDoc);
+                            lastDoc = tds.doc();
+                        }
+                        else
+                        {
+                            assertFalse(tds.next());
+                        }
+                    }
+                    for (int j = 0; j < update.length; j++)
+                    {
+                        TermDocs tds = reader.termDocs(new Term("TEXT", update[j]));
+                        if (j < i)
+                        {
+                            assertTrue(tds.next());
+                            assertTrue(tds.doc() > lastDoc);
+                            lastDoc = tds.doc();
+                        }
+                        else
+                        {
+                            assertFalse(tds.next());
+                        }
+                    }
+                    reader.close();
+
+                    reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions);
+                    
+                    lastDoc = -1;
+                    for (int j = 0; j < create.length; j++)
+                    {
+                        TermDocs tds = reader.termDocs(new Term("TEXT", create[j]));
+                        if (j > i)
+                        {
+                            assertTrue(tds.next());
+                            assertTrue(tds.doc() > lastDoc);
+                            lastDoc = tds.doc();
+                        }
+                        else
+                        {
+                            assertFalse(tds.next());
+                        }
+                    }
+                    for (int j = 0; j < update.length; j++)
+                    {
+                        TermDocs tds = reader.termDocs(new Term("TEXT", update[j]));
+                        if (j <= i)
+                        {
+                            assertTrue(tds.next());
+                            assertTrue(tds.doc() > lastDoc);
+                            lastDoc = tds.doc();
+                        }
+                        else
+                        {
+                            assertFalse(tds.next());
+                        }
+                    }
+
+                    reader.close();
+
+                    ii.setStatus(guid, TransactionStatus.COMMITTING, null, null);
+                    ii.setStatus(guid, TransactionStatus.COMMITTED, null, null);
+
+                    reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader();
+                   
+                    lastDoc = -1;
+                    for (int j = 0; j < create.length; j++)
+                    {
+                        TermDocs tds = reader.termDocs(new Term("TEXT", create[j]));
+                        if (j > i)
+                        {
+                            assertTrue(tds.next());
+                            assertTrue(tds.doc() > lastDoc);
+                            lastDoc = tds.doc();
+                        }
+                        else
+                        {
+                            assertFalse(tds.next());
+                        }
+                    }
+                    for (int j = 0; j < update.length; j++)
+                    {
+                        TermDocs tds = reader.termDocs(new Term("TEXT", update[j]));
+                        if (j <= i)
+                        {
+                            assertTrue(tds.next());
+                            assertTrue(tds.doc() > lastDoc);
+                            lastDoc = tds.doc();
+                        }
+                        else
+                        {
+                            assertFalse(tds.next());
+                        }
+                    }
+                    reader.close();
+
+                }
+
+            }
+            catch (IOException e)
+            {
+                System.exit(-1);
+            }
+        }
+
+    }
+}

source/java/org/alfresco/repo/search/impl/lucene/index/IndexType.java

@@ -12,30 +12,19 @@ public enum IndexType
      */
     INDEX,
     
-    /**
-     * Identifies a subindex. This will also be optimised. Sub-indexes are periodically merged into the index.
-     */
-    SUBINDEX,
-    
     /**
      * An overlay. This is an optimised index with a deletion list. To commit an overlay requires no deletions against other indexes. Deletions are done when an overlay turns
-     * into or is merged into a subindex. Overlays are periodically merged into a sub index. An overlay can require or have background properties indexed
+     * into or is merged into a index. Overlays are periodically merged into an index. An overlay can require or have background properties indexed.
-     */
-    INDEX_OVERLAY,
-    
-    /**
-     * A long running overlay defintion against the index. Not yet supported.
-     * This, itself, may have transactional additions. 
-     */
-    OVERLAY,
-    
-    /**
-     * A delta is a transactional change set. This commits to an overlay index.
      */
     DELTA,
     
     /**
-     * A delta to an overlay
+     * A long running overlay definition against the index. Not yet supported.
+     * This, itself, may have transactional additions. 
      */
+    OVERLAY,
+ 
     OVERLAY_DELTA;
+    
+    
 }

source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCounting.java

@@ -24,7 +24,7 @@ public interface ReferenceCounting
 
     public void decrementReferenceCount() throws IOException;
 
-    public boolean isUsed();
+    public int getReferenceCount();
 
     public void setInvalidForReuse() throws IOException;
 }

source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java

@@ -18,70 +18,52 @@ package org.alfresco.repo.search.impl.lucene.index;
 
 import java.io.IOException;
 
-import org.aopalliance.intercept.MethodInvocation;
+import org.apache.log4j.Logger;
+import org.apache.lucene.index.FilterIndexReader;
 import org.apache.lucene.index.IndexReader;
-import org.springframework.aop.framework.ProxyFactory;
-import org.springframework.aop.framework.adapter.AdvisorAdapterRegistry;
-import org.springframework.aop.framework.adapter.GlobalAdvisorAdapterRegistry;
-import org.springframework.aop.support.DelegatingIntroductionInterceptor;
-import org.springframework.aop.target.SingletonTargetSource;
 
 public class ReferenceCountingReadOnlyIndexReaderFactory
 {
-    public static IndexReader createReader(IndexReader indexReader)
+    public static IndexReader createReader(String id, IndexReader indexReader)
     {
-        AdvisorAdapterRegistry advisorAdapterRegistry = GlobalAdvisorAdapterRegistry.getInstance();
+        return new ReferenceCountingReadOnlyIndexReader(id, indexReader);
-        ProxyFactory proxyFactory = new ProxyFactory();
-        proxyFactory.addAdvisor(advisorAdapterRegistry.wrap(new Interceptor(indexReader)));
-        proxyFactory.setTargetSource(new SingletonTargetSource(indexReader));
-        IndexReader proxy = (IndexReader) proxyFactory.getProxy();
-        return proxy;
     }
 
-    public static class Interceptor extends DelegatingIntroductionInterceptor implements ReferenceCounting
+    public static class ReferenceCountingReadOnlyIndexReader extends FilterIndexReader implements ReferenceCounting
     {
+        private static Logger s_logger = Logger.getLogger(ReferenceCountingReadOnlyIndexReader.class);
+        
         
         private static final long serialVersionUID = 7693185658022810428L;
 
-        IndexReader indexReader;
+        String id;
-
+     
         int refCount = 0;
 
         boolean invalidForReuse = false;
 
-        Interceptor(IndexReader indexReader)
+        ReferenceCountingReadOnlyIndexReader(String id, IndexReader indexReader)
         {
-            this.indexReader = indexReader;
+            super(indexReader);
-        }
+            this.id = id;
-
-        public Object invoke(MethodInvocation mi) throws Throwable
-        {
-            // Read only
-            String methodName = mi.getMethod().getName();
-            if (methodName.equals("delete") || methodName.equals("doDelete"))
-            {
-                throw new UnsupportedOperationException("Delete is not supported by read only index readers");
-            }
-            // Close
-            else if (methodName.equals("close"))
-            {
-                decrementReferenceCount();
-                return null;
-            }
-            else
-            {
-                return super.invoke(mi);
-            }
         }
 
         public synchronized void incrementReferenceCount()
         {
             refCount++;
+            if(s_logger.isDebugEnabled())
+            {
+                s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " - increment - ref count is "+refCount);
+            }
         }
 
         public synchronized void decrementReferenceCount() throws IOException
         {
             refCount--;
+            if(s_logger.isDebugEnabled())
+            {
+                s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " - decrement - ref count is "+refCount);
+            }
             closeIfRequired();
         }
 
@@ -89,19 +71,52 @@ public class ReferenceCountingReadOnlyIndexReaderFactory
         {
             if ((refCount == 0) && invalidForReuse)
             {
-                indexReader.close();
+                if(s_logger.isDebugEnabled())
+                {
+                    s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " closed.");
+                }
+                in.close();
+            }
+            else
+            {
+                if(s_logger.isDebugEnabled())
+                {
+                    s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " still open .... ref = "+refCount+" invalidForReuse = "+invalidForReuse);
+                }
             }
         }
 
-        public synchronized boolean isUsed()
+        public synchronized int getReferenceCount()
         {
-            return (refCount > 0);
+            return refCount;
         }
 
         public synchronized void setInvalidForReuse() throws IOException
         {
             invalidForReuse = true;
+            if(s_logger.isDebugEnabled())
+            {
+                s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " set invalid for reuse");
+            }
             closeIfRequired();
         }
+
+        @Override
+        protected void doClose() throws IOException
+        {
+            if(s_logger.isDebugEnabled())
+            {
+                s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " closing");
+            }
+            decrementReferenceCount();
+        }
+
+        @Override
+        protected void doDelete(int n) throws IOException
+        {
+            throw new UnsupportedOperationException("Delete is not supported by read only index readers");
+        }
+        
+        
     }
 }

source/java/org/alfresco/repo/search/impl/lucene/index/TransactionStatus.java

@@ -21,17 +21,48 @@ package org.alfresco.repo.search.impl.lucene.index;
 
 public enum TransactionStatus
 {
+    
     // Match the order in javax.transaction.Status so ordinal values are correct
     ACTIVE
     {
-        public boolean follows(TransactionStatus previous)
+        public boolean isCommitted()
         {
             return false;
         }
+        
+        public boolean isTransient()
+        {
+            return true;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return true;
+        }
+        
+        public boolean follows(TransactionStatus previous)
+        {
+            return previous == null;
+        }
     },
 
     MARKED_ROLLBACK
     {
+        public boolean isCommitted()
+        {
+            return false;
+        }
+        
+        public boolean isTransient()
+        {
+            return true;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return true;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return previous.allowsRollbackOrMark(previous);
@@ -40,6 +71,21 @@ public enum TransactionStatus
 
     PREPARED
     {
+        public boolean isCommitted()
+        {
+            return false;
+        }
+        
+        public boolean isTransient()
+        {
+            return false;
+        }
+
+        public boolean canBeReordered()
+        {
+            return false;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return previous == TransactionStatus.PREPARING;
@@ -53,6 +99,16 @@ public enum TransactionStatus
             return true;
         }
 
+        public boolean isTransient()
+        {
+            return false;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return false;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return previous == TransactionStatus.COMMITTING;
@@ -61,6 +117,21 @@ public enum TransactionStatus
 
     ROLLEDBACK
     {
+        public boolean isCommitted()
+        {
+            return false;
+        }
+        
+        public boolean isTransient()
+        {
+            return true;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return true;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return previous == TransactionStatus.ROLLINGBACK;
@@ -69,6 +140,21 @@ public enum TransactionStatus
 
     UNKNOWN
     {
+        public boolean isCommitted()
+        {
+            return false;
+        }
+        
+        public boolean isTransient()
+        {
+            return true;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return true;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return false;
@@ -77,6 +163,21 @@ public enum TransactionStatus
 
     NO_TRANSACTION
     {
+        public boolean isCommitted()
+        {
+            return false;
+        }
+        
+        public boolean isTransient()
+        {
+            return true;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return true;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return false;
@@ -85,6 +186,21 @@ public enum TransactionStatus
 
     PREPARING
     {
+        public boolean isCommitted()
+        {
+            return false;
+        }
+        
+        public boolean isTransient()
+        {
+            return true;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return true;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return previous == TransactionStatus.ACTIVE;
@@ -93,6 +209,21 @@ public enum TransactionStatus
 
     COMMITTING
     {
+        public boolean isCommitted()
+        {
+            return false;
+        }
+        
+        public boolean isTransient()
+        {
+            return true;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return false;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return previous == TransactionStatus.PREPARED;
@@ -101,6 +232,21 @@ public enum TransactionStatus
 
     ROLLINGBACK
     {
+        public boolean isCommitted()
+        {
+            return false;
+        }
+
+        public boolean isTransient()
+        {
+            return true;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return true;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return previous.allowsRollbackOrMark(previous);
@@ -117,6 +263,16 @@ public enum TransactionStatus
             return true;
         }
 
+        public boolean isTransient()
+        {
+            return false;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return false;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return false;
@@ -128,6 +284,21 @@ public enum TransactionStatus
      */
     MERGE_TARGET
     {
+        public boolean isCommitted()
+        {
+            return false;
+        }
+        
+        public boolean isTransient()
+        {
+            return false;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return false;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return false;
@@ -137,67 +308,108 @@ public enum TransactionStatus
     /*
      * These index overlays require reindexing
      */
-    COMMITTED_REQUIRES_REINDEX
+//    COMMITTED_REQUIRES_REINDEX
-    {
+//    {
-        public boolean isCommitted()
+//        public boolean isCommitted()
-        {
+//        {
-            return true;
+//            return true;
-        }
+//        }
-
+//
-        public boolean follows(TransactionStatus previous)
+//        public boolean isTransient()
-        {
+//        {
-            return false;
+//            return false;
-        }
+//        }
-    },
+//        
+//        public boolean canBeReordered()
+//        {
+//            return false;
+//        }
+//        
+//        public boolean follows(TransactionStatus previous)
+//        {
+//            return false;
+//        }
+//    },
 
     /*
      * These index overlays are reindexing
      */
-    COMMITTED_REINDEXING
+//    COMMITTED_REINDEXING
-    {
+//    {
-        public boolean isCommitted()
+//        public boolean isCommitted()
-        {
+//        {
-            return true;
+//            return true;
-        }
+//        }
-
+//
-        public boolean follows(TransactionStatus previous)
+//        
-        {
+//        public boolean canBeReordered()
-            return false;
+//        {
-        }
+//            return false;
-    },
+//        }
+//        
+//        public boolean isTransient()
+//        {
+//            return false;
+//        }
+//        
+//        public boolean follows(TransactionStatus previous)
+//        {
+//            return false;
+//        }
+//    },
 
     /*
      * These index overlays have ben reindexed.
      */
-    COMMITTED_REINDEXED
+//    COMMITTED_REINDEXED
-    {
+//    {
-        public boolean isCommitted()
+//        public boolean isCommitted()
-        {
+//        {
-            return true;
+//            return true;
-        }
+//        }
-
+//
-        public boolean follows(TransactionStatus previous)
+//        public boolean isTransient()
-        {
+//        {
-            return false;
+//            return false;
-        }
+//        }
-    },
+//        
+//        public boolean canBeReordered()
+//        {
+//            return false;
+//        }
+//        
+//        public boolean follows(TransactionStatus previous)
+//        {
+//            return false;
+//        }
+//    },
 
     /*
      * Committed but the index still has deletions
      */
 
-    COMMITTED_WITH_DELETIONS
+//    COMMITTED_WITH_DELETIONS
-    {
+//    {
-        public boolean isCommitted()
+//        public boolean isCommitted()
-        {
+//        {
-            return true;
+//            return true;
-        }
+//        }
-
+//
-        public boolean follows(TransactionStatus previous)
+//        public boolean isTransient()
-        {
+//        {
-            return false;
+//            return false;
-        }
+//        }
-    },
+//        
+//        public boolean canBeReordered()
+//        {
+//            return false;
+//        }
+//        
+//        public boolean follows(TransactionStatus previous)
+//        {
+//            return false;
+//        }
+//    },
 
     /*
      * Pending deleted are being committed to for the delta.
@@ -209,6 +421,16 @@ public enum TransactionStatus
             return true;
         }
 
+        public boolean isTransient()
+        {
+            return false;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return false;
+        }
+        
         public boolean follows(TransactionStatus previous)
         {
             return false;
@@ -220,16 +442,32 @@ public enum TransactionStatus
      */
     DELETABLE
     {
-        public boolean follows(TransactionStatus previous)
+        public boolean isCommitted()
         {
             return false;
         }
+        
+        public boolean isTransient()
+        {
+            return false;
+        }
+        
+        public boolean canBeReordered()
+        {
+            return false;
+        }
+        
+        public boolean follows(TransactionStatus previous)
+        {
+            return true;
+        }
     };
 
-    public boolean isCommitted()
+    public abstract boolean isCommitted();
-    {
+    
-        return false;
+    public abstract boolean isTransient();
-    }
+    
+    public abstract boolean canBeReordered();
 
     public abstract boolean follows(TransactionStatus previous);