Fix reference counting in the new indexer.

Added instrumentation to the new indexer.
Exposed lots of configuration options for the new indexer.
Added debug settings for the new indexer.
Fixed search test model for sort tests (1.3 merge fun)

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@3455 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/repo/search/impl/lucene/LuceneTest2.java

@@ -41,6 +41,7 @@ import org.alfresco.repo.dictionary.M2Model;
 import org.alfresco.repo.node.BaseNodeServiceTest;
 import org.alfresco.repo.search.QueryParameterDefImpl;
 import org.alfresco.repo.search.QueryRegisterComponent;
+import org.alfresco.repo.search.impl.lucene.analysis.NumericEncoder;
 import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer;
 import org.alfresco.repo.search.results.ChildAssocRefResultSet;
 import org.alfresco.repo.search.results.DetachedResultSet;
@@ -371,6 +372,7 @@ public class LuceneTest2 extends TestCase
         super(arg0);
     }
 
+    
     public void test0() throws Exception
     {
         luceneFTS.pause();

source/java/org/alfresco/repo/search/impl/lucene/LuceneTest_model.xml

@@ -18,6 +18,58 @@
         <type name="test:testSuperType">
             <title>Test Super Type</title>
             <parent>sys:container</parent>
+            <properties>
+                <property name="test:createdDate">
+                    <type>d:date</type>
+                    <mandatory>true</mandatory>
+                    <multiple>false</multiple>
+                    <index enabled="true">
+                        <atomic>true</atomic>
+                        <stored>true</stored>
+                        <tokenised>true</tokenised>
+                    </index>
+                </property>
+                <property name="test:orderDouble">
+                    <type>d:double</type>
+                    <mandatory>true</mandatory>
+                    <multiple>false</multiple>
+                    <index enabled="true">
+                        <atomic>true</atomic>
+                        <stored>true</stored>
+                        <tokenised>true</tokenised>
+                    </index>
+                </property>
+                <property name="test:orderFloat">
+                    <type>d:float</type>
+                    <mandatory>true</mandatory>
+                    <multiple>false</multiple>
+                    <index enabled="true">
+                        <atomic>true</atomic>
+                        <stored>true</stored>
+                        <tokenised>true</tokenised>
+                    </index>
+                </property>
+                <property name="test:orderLong">
+                    <type>d:long</type>
+                    <mandatory>true</mandatory>
+                    <multiple>false</multiple>
+                    <index enabled="true">
+                        <atomic>true</atomic>
+                        <stored>true</stored>
+                        <tokenised>true</tokenised>
+                    </index>
+                </property>
+                <property name="test:orderInt">
+                    <type>d:int</type>
+                    <mandatory>true</mandatory>
+                    <multiple>false</multiple>
+                    <index enabled="true">
+                        <atomic>true</atomic>
+                        <stored>true</stored>
+                        <tokenised>true</tokenised>
+                    </index>
+                </property>
+            </properties>
             <associations>
                 <child-association name="test:assoc">
                     <source>
@@ -183,7 +235,6 @@
             </mandatory-aspects>
         </type>
     </types>
->>>>>>> .merge-right.r3203
    
    <aspects>
       <aspect name="test:testSuperAspect">

source/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java

@@ -154,6 +154,7 @@ class IndexEntry
         builder.append("Type=").append(getType()).append(" ");
         builder.append("Status=").append(getStatus()).append(" ");
         builder.append("Docs=").append(getDocumentCount()).append(" ");
+        builder.append("Deletions=").append(getDeletions()).append(" ");
         return builder.toString();
     }
     

source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java

@@ -49,13 +49,11 @@ import org.alfresco.error.AlfrescoRuntimeException;
 import org.alfresco.repo.search.IndexerException;
 import org.alfresco.repo.search.impl.lucene.FilterIndexReaderByNodeRefs2;
 import org.alfresco.service.cmr.repository.NodeRef;
-import org.alfresco.service.cmr.repository.StoreRef;
 import org.alfresco.util.GUID;
 import org.apache.log4j.Logger;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.MultiReader;
@@ -206,6 +204,32 @@ public class IndexInfo
 
     private static HashMap<File, IndexInfo> indexInfos = new HashMap<File, IndexInfo>();
 
+    private int maxDocsForInMemoryMerge = 10000;
+
+    private int writerMinMergeDocs = 1000;
+
+    private int writerMergeFactor = 5;
+
+    private int writerMaxMergeDocs = 1000000;
+
+    private boolean writerUseCompoundFile = true;
+
+    private int mergerMinMergeDocs = 1000;
+
+    private int mergerMergeFactor = 5;
+
+    private int mergerMaxMergeDocs = 1000000;
+
+    private boolean mergerUseCompoundFile = true;
+
+    private int mergerTargetOverlays = 5;
+
+    // TODO: Something to control the maximum number of overlays
+
+    private boolean enableCleanerThread = true;
+
+    private boolean enableMergerThread = true;
+
     static
     {
         System.setProperty("disableLuceneLocks", "true");
@@ -283,6 +307,10 @@ public class IndexInfo
                             try
                             {
                                 writer = new IndexWriter(oldIndex, new StandardAnalyzer(), false);
+                                writer.setUseCompoundFile(writerUseCompoundFile);
+                                writer.minMergeDocs = writerMinMergeDocs;
+                                writer.mergeFactor = writerMergeFactor;
+                                writer.maxMergeDocs = writerMaxMergeDocs;
                                 writer.optimize();
                                 long docs = writer.docCount();
                                 writer.close();
@@ -393,24 +421,32 @@ public class IndexInfo
             }
         }
         // TODO: Add unrecognised folders for deletion.
+
+        if (enableCleanerThread)
+        {
             cleanerThread = new Thread(cleaner);
             cleanerThread.setDaemon(true);
-        cleanerThread.setName("Index cleaner thread");
+            cleanerThread.setName("Index cleaner thread "+indexDirectory);
             cleanerThread.start();
+        }
 
+       
+        if (enableMergerThread)
+        {
             mergerThread = new Thread(merger);
             mergerThread.setDaemon(true);
-        mergerThread.setName("Index merger thread");
+            mergerThread.setName("Index merger thread "+indexDirectory);
             mergerThread.start();
+        }
 
         IndexWriter writer;
         try
         {
             writer = new IndexWriter(emptyIndex, new StandardAnalyzer(), true);
-            writer.setUseCompoundFile(true);
+            writer.setUseCompoundFile(writerUseCompoundFile);
-            writer.minMergeDocs = 1000;
+            writer.minMergeDocs = writerMinMergeDocs;
-            writer.mergeFactor = 5;
+            writer.mergeFactor = writerMergeFactor;
-            writer.maxMergeDocs = 1000000;
+            writer.maxMergeDocs = writerMaxMergeDocs;
         }
         catch (IOException e)
         {
@@ -441,6 +477,15 @@ public class IndexInfo
             // close index writer if required
             closeDeltaIndexWriter(id);
             // Check the index knows about the transaction
+            reader = buildAndRegisterDeltaReader(id);
+            indexReaders.put(id, reader);
+        }
+        return reader;
+    }
+
+    private IndexReader buildAndRegisterDeltaReader(String id) throws IOException
+    {
+        IndexReader reader;
         File location = ensureDeltaIsRegistered(id);
         // Create a dummy index reader to deal with empty indexes and not persist these.
         if (IndexReader.indexExists(location))
@@ -451,8 +496,6 @@ public class IndexInfo
         {
             reader = IndexReader.open(emptyIndex);
         }
-            indexReaders.put(id, reader);
-        }
         return reader;
     }
 
@@ -511,10 +554,10 @@ public class IndexInfo
         if (!IndexReader.indexExists(location))
         {
             IndexWriter creator = new IndexWriter(location, analyzer, true);
-            creator.setUseCompoundFile(true);
+            creator.setUseCompoundFile(writerUseCompoundFile);
-            creator.minMergeDocs = 1000;
+            creator.minMergeDocs = writerMinMergeDocs;
-            creator.mergeFactor = 5;
+            creator.mergeFactor = writerMergeFactor;
-            creator.maxMergeDocs = 1000000;
+            creator.maxMergeDocs = writerMaxMergeDocs;
             return creator;
         }
         return null;
@@ -538,10 +581,10 @@ public class IndexInfo
             if (writer == null)
             {
                 writer = new IndexWriter(location, analyzer, false);
-                writer.setUseCompoundFile(true);
+                writer.setUseCompoundFile(writerUseCompoundFile);
-                writer.minMergeDocs = 1000;
+                writer.minMergeDocs = writerMinMergeDocs;
-                writer.mergeFactor = 5;
+                writer.mergeFactor = writerMergeFactor;
-                writer.maxMergeDocs = 1000000;
+                writer.maxMergeDocs = writerMaxMergeDocs;
             }
             indexWriters.put(id, writer);
         }
@@ -789,12 +832,14 @@ public class IndexInfo
             // TODO: Should use the in memory index but we often end up forcing to disk anyway.
             // Is it worth it?
             // luceneIndexer.flushPending();
-            IndexReader deltaReader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(id,
+            
-                    getDeltaIndexReader(id));
+            IndexReader deltaReader =  buildAndRegisterDeltaReader(id);
-            ReferenceCounting deltaRefCount = (ReferenceCounting) deltaReader;
-            deltaRefCount.incrementReferenceCount();
             IndexReader reader = new MultiReader(new IndexReader[] {
                     new FilterIndexReaderByNodeRefs2(mainIndexReader, deletions, deleteOnlyNodes), deltaReader });
+            reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader("MainReader"+id, reader);
+            ReferenceCounting refCounting = (ReferenceCounting)reader;
+            refCounting.incrementReferenceCount();
+            refCounting.setInvalidForReuse();
             return reader;
         }
         finally
@@ -1659,7 +1704,18 @@ public class IndexInfo
         {
             if (indexIsShared)
             {
+                long start = 0l;
+                if (s_logger.isDebugEnabled())
+                {
+                    s_logger.debug(" ... waiting for file lock");
+                    start = System.nanoTime();
+                }
                 fileLock = indexInfoChannel.lock();
+                if (s_logger.isDebugEnabled())
+                {
+                    long end = System.nanoTime();
+                    s_logger.debug(" ... got file lock in " + ((end - start)/10e6f) + " ms");
+                }
                 if (!checkVersion())
                 {
                     setStatusFromFile();
@@ -1688,6 +1744,10 @@ public class IndexInfo
                 try
                 {
                     fileLock.release();
+                    if (s_logger.isDebugEnabled())
+                    {
+                        s_logger.debug(" ... released file lock");
+                    }
                 }
                 catch (IOException e)
                 {
@@ -1696,6 +1756,11 @@ public class IndexInfo
         }
     }
 
+    /**
+     * Helper to print out index information
+     * 
+     * @param args
+     */
     public static void main(String[] args)
     {
 
@@ -1721,67 +1786,6 @@ public class IndexInfo
         }
     }
 
-    // public static void main(String[] args) throws IOException
-
-    // {
-    // System.setProperty("disableLuceneLocks", "true");
-    //
-    // HashSet<NodeRef> deletions = new HashSet<NodeRef>();
-    // for (int i = 0; i < 0; i++)
-    // {
-    // deletions.add(new NodeRef(new StoreRef("woof", "bingle"), GUID.generate()));
-    // }
-    //
-    // int repeat = 100;
-    // int docs = 1;
-    // final IndexInfo ii = new IndexInfo(new File("c:\\indexTest"));
-    //
-    // long totalTimeA = 0;
-    // long countA = 0;
-    //
-    // while (true)
-    // {
-    // long start = System.nanoTime();
-    // for (int i = 0; i < repeat; i++)
-    // {
-    // String guid = GUID.generate();
-    // ii.setStatus(guid, TransactionStatus.ACTIVE, null, null);
-    // IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer());
-    //
-    // for (int j = 0; j < docs; j++)
-    // {
-    // Document doc = new Document();
-    // for (int k = 0; k < 15; k++)
-    // {
-    // doc.add(new Field("ID" + k, guid + " " + j + " " + k, false, true, false));
-    // }
-    // writer.addDocument(doc);
-    // }
-    //
-    // ii.closeDeltaIndexWriter(guid);
-    // ii.setStatus(guid, TransactionStatus.PREPARING, null, null);
-    // ii.setPreparedState(guid, deletions, docs, false);
-    // ii.getDeletions(guid);
-    // ii.setStatus(guid, TransactionStatus.PREPARED, null, null);
-    // ii.setStatus(guid, TransactionStatus.COMMITTING, null, null);
-    // ii.setStatus(guid, TransactionStatus.COMMITTED, null, null);
-    // for (int j = 0; j < 0; j++)
-    // {
-    // ii.getMainIndexReferenceCountingReadOnlyIndexReader();
-    // }
-    // }
-    //
-    // long end = System.nanoTime();
-    //
-    // totalTimeA += (end - start);
-    // countA += repeat;
-    // float average = countA * 1000000000f / totalTimeA;
-    //
-    // System.out.println("Repeated "
-    // + repeat + " in " + ((end - start) / 1000000000.0) + " average = " + average);
-    // }
-    // }
-
     /**
      * Clean up support.
      * 
@@ -1979,7 +1983,7 @@ public class IndexInfo
                         if (!mergingIndexes && !applyingDeletions)
                         {
 
-                            if ((indexes > 5) || (deltas > 5))
+                            if ((indexes > mergerMergeFactor) || (deltas > mergerTargetOverlays))
                             {
                                 if (indexes > deltas)
                                 {
@@ -2333,7 +2337,7 @@ public class IndexInfo
                             }
                         }
 
-                        int position = findMergeIndex(1, 1000000, 5, mergeList);
+                        int position = findMergeIndex(1, mergerMaxMergeDocs, mergerMergeFactor, mergeList);
                         String firstMergeId = mergeList.get(position).getName();
 
                         long count = 0;
@@ -2417,7 +2421,7 @@ public class IndexInfo
                         else if (entry.getStatus() == TransactionStatus.MERGE_TARGET)
                         {
                             outputLocation = location;
-                            if (docCount < 10000)
+                            if (docCount < maxDocsForInMemoryMerge)
                             {
                                 ramDirectory = new RAMDirectory();
                                 writer = new IndexWriter(ramDirectory, new StandardAnalyzer(), true);
@@ -2425,11 +2429,12 @@ public class IndexInfo
                             else
                             {
                                 writer = new IndexWriter(location, new StandardAnalyzer(), true);
+                               
                             }
-                            writer.setUseCompoundFile(true);
+                            writer.setUseCompoundFile(mergerUseCompoundFile);
-                            writer.minMergeDocs = 1000;
+                            writer.minMergeDocs = mergerMinMergeDocs;
-                            writer.mergeFactor = 5;
+                            writer.mergeFactor = mergerMergeFactor;
-                            writer.maxMergeDocs = 1000000;
+                            writer.maxMergeDocs = mergerMaxMergeDocs;
                         }
                     }
                     writer.addIndexes(readers);
@@ -2525,10 +2530,6 @@ public class IndexInfo
                             indexEntries.remove(id);
                             deleteQueue.add(id);
                         }
-                        synchronized (cleaner)
-                        {
-                            cleaner.notify();
-                        }
                         
                         dumpInfo();
 
@@ -2536,6 +2537,11 @@ public class IndexInfo
 
                         clearOldReaders();
 
+                        synchronized (cleaner)
+                        {
+                            cleaner.notify();
+                        }
+                        
                         return null;
                     }
 
@@ -2603,10 +2609,19 @@ public class IndexInfo
 
     private void getWriteLock()
     {
+        String threadName = null;
+        long start = 0l;
+        if (s_logger.isDebugEnabled())
+        {
+            threadName = Thread.currentThread().getName();
+            s_logger.debug("Waiting for WRITE lock  - " + threadName);
+            start = System.nanoTime();
+        }
         readWriteLock.writeLock().lock();
         if (s_logger.isDebugEnabled())
         {
-            s_logger.debug("GOT WRITE LOCK  - " + Thread.currentThread().getName());
+            long end = System.nanoTime();
+            s_logger.debug("...GOT WRITE LOCK  - " + threadName + " -  in " + ((end - start)/10e6f) + " ms");
         }
     }
 
@@ -2614,17 +2629,26 @@ public class IndexInfo
     {
         if (s_logger.isDebugEnabled())
         {
-            s_logger.debug("RELEASES WRITE LOCK  - " + Thread.currentThread().getName());
+            s_logger.debug("RELEASED WRITE LOCK  - " + Thread.currentThread().getName());
         }
         readWriteLock.writeLock().unlock();
     }
 
     private void getReadLock()
     {
+        String threadName = null;
+        long start = 0l;
+        if (s_logger.isDebugEnabled())
+        {
+            threadName = Thread.currentThread().getName();
+            s_logger.debug("Waiting for READ lock  - " + threadName);
+            start = System.nanoTime();
+        }
         readWriteLock.readLock().lock();
         if (s_logger.isDebugEnabled())
         {
-            s_logger.debug("GOT READ LOCK  - " + Thread.currentThread().getName());
+            long end = System.nanoTime();
+            s_logger.debug("...GOT READ LOCK  - " + threadName + " -  in " + ((end - start)/10e6f) + " ms");
         }
     }
 
@@ -2632,7 +2656,7 @@ public class IndexInfo
     {
         if (s_logger.isDebugEnabled())
         {
-            s_logger.debug("RELEASES READ LOCK  - " + Thread.currentThread().getName());
+            s_logger.debug("RELEASED READ LOCK  - " + Thread.currentThread().getName());
         }
         readWriteLock.readLock().unlock();
     }
@@ -2641,4 +2665,136 @@ public class IndexInfo
     {
         return indexDirectory.toString();
     }
+
+    public boolean isEnableCleanerThread()
+    {
+        return enableCleanerThread;
+    }
+
+    public void setEnableCleanerThread(boolean enableCleanerThread)
+    {
+        this.enableCleanerThread = enableCleanerThread;
+    }
+
+    public boolean isEnableMergerThread()
+    {
+        return enableMergerThread;
+    }
+
+    public void setEnableMergerThread(boolean enableMergerThread)
+    {
+        this.enableMergerThread = enableMergerThread;
+    }
+
+    public boolean isIndexIsShared()
+    {
+        return indexIsShared;
+    }
+
+    public void setIndexIsShared(boolean indexIsShared)
+    {
+        this.indexIsShared = indexIsShared;
+    }
+
+    public int getMaxDocsForInMemoryMerge()
+    {
+        return maxDocsForInMemoryMerge;
+    }
+
+    public void setMaxDocsForInMemoryMerge(int maxDocsForInMemoryMerge)
+    {
+        this.maxDocsForInMemoryMerge = maxDocsForInMemoryMerge;
+    }
+
+    public int getMergerMaxMergeDocs()
+    {
+        return mergerMaxMergeDocs;
+    }
+
+    public void setMergerMaxMergeDocs(int mergerMaxMergeDocs)
+    {
+        this.mergerMaxMergeDocs = mergerMaxMergeDocs;
+    }
+
+    public int getMergerMergeFactor()
+    {
+        return mergerMergeFactor;
+    }
+
+    public void setMergerMergeFactor(int mergerMergeFactor)
+    {
+        this.mergerMergeFactor = mergerMergeFactor;
+    }
+
+    public int getMergerMinMergeDocs()
+    {
+        return mergerMinMergeDocs;
+    }
+
+    public void setMergerMinMergeDocs(int mergerMinMergeDocs)
+    {
+        this.mergerMinMergeDocs = mergerMinMergeDocs;
+    }
+
+    public int getMergerTargetOverlays()
+    {
+        return mergerTargetOverlays;
+    }
+
+    public void setMergerTargetOverlays(int mergerTargetOverlays)
+    {
+        this.mergerTargetOverlays = mergerTargetOverlays;
+    }
+
+    public boolean isMergerUseCompoundFile()
+    {
+        return mergerUseCompoundFile;
+    }
+
+    public void setMergerUseCompoundFile(boolean mergerUseCompoundFile)
+    {
+        this.mergerUseCompoundFile = mergerUseCompoundFile;
+    }
+
+    public int getWriterMaxMergeDocs()
+    {
+        return writerMaxMergeDocs;
+    }
+
+    public void setWriterMaxMergeDocs(int writerMaxMergeDocs)
+    {
+        this.writerMaxMergeDocs = writerMaxMergeDocs;
+    }
+
+    public int getWriterMergeFactor()
+    {
+        return writerMergeFactor;
+    }
+
+    public void setWriterMergeFactor(int writerMergeFactor)
+    {
+        this.writerMergeFactor = writerMergeFactor;
+    }
+
+    public int getWriterMinMergeDocs()
+    {
+        return writerMinMergeDocs;
+    }
+
+    public void setWriterMinMergeDocs(int writerMinMergeDocs)
+    {
+        this.writerMinMergeDocs = writerMinMergeDocs;
+    }
+
+    public boolean isWriterUseCompoundFile()
+    {
+        return writerUseCompoundFile;
+    }
+
+    public void setWriterUseCompoundFile(boolean writerUseCompoundFile)
+    {
+        this.writerUseCompoundFile = writerUseCompoundFile;
+    }
+    
+    
 }

source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfoTest.java

@@ -207,6 +207,7 @@ public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", "
             writer.addDocument(doc);
 
             ii.closeDeltaIndexWriter(guid);
+ 
             ii.setStatus(guid, TransactionStatus.PREPARING, null, null);
             ii.setPreparedState(guid, new HashSet<NodeRef>(), 1, false);
             ii.getDeletions(guid);

source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java

@@ -33,7 +33,6 @@ public class ReferenceCountingReadOnlyIndexReaderFactory
     {
         private static Logger s_logger = Logger.getLogger(ReferenceCountingReadOnlyIndexReader.class);
 
-        
         private static final long serialVersionUID = 7693185658022810428L;
 
         String id;
@@ -42,6 +41,8 @@ public class ReferenceCountingReadOnlyIndexReaderFactory
 
         boolean invalidForReuse = false;
 
+        boolean allowsDeletions;
+
         ReferenceCountingReadOnlyIndexReader(String id, IndexReader indexReader)
         {
             super(indexReader);
@@ -53,7 +54,8 @@ public class ReferenceCountingReadOnlyIndexReaderFactory
             refCount++;
             if (s_logger.isDebugEnabled())
             {
-                s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " - increment - ref count is "+refCount);
+                s_logger.debug(Thread.currentThread().getName()
+                        + ": Reader " + id + " - increment - ref count is " + refCount);
             }
         }
 
@@ -62,7 +64,8 @@ public class ReferenceCountingReadOnlyIndexReaderFactory
             refCount--;
             if (s_logger.isDebugEnabled())
             {
-                s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " - decrement - ref count is "+refCount);
+                s_logger.debug(Thread.currentThread().getName()
+                        + ": Reader " + id + " - decrement - ref count is " + refCount);
             }
             closeIfRequired();
         }
@@ -81,7 +84,9 @@ public class ReferenceCountingReadOnlyIndexReaderFactory
             {
                 if (s_logger.isDebugEnabled())
                 {
-                    s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " still open .... ref = "+refCount+" invalidForReuse = "+invalidForReuse);
+                    s_logger.debug(Thread.currentThread().getName()
+                            + ": Reader " + id + " still open .... ref = " + refCount + " invalidForReuse = "
+                            + invalidForReuse);
                 }
             }
         }
@@ -117,6 +122,5 @@ public class ReferenceCountingReadOnlyIndexReaderFactory
             throw new UnsupportedOperationException("Delete is not supported by read only index readers");
         }
 
-        
     }
 }