Asynchronous indexing for AVM

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@5795 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/public-services-context.xml

@@ -892,6 +892,23 @@
         <property name="enableIndexing">
             <value>true</value>
         </property>
+		<property name="defaultMode">
+            <value>SYNCHRONOUS</value>
+        </property>
+		<property name="indexingDefinitions">
+			<list>
+				<value>SYNCHRONOUS:TYPE:STAGING</value>
+				<value>UNINDEXED:TYPE:STAGING_PREVIEW</value>
+				<value>UNINDEXED:TYPE:AUTHOR</value>
+				<value>UNINDEXED:TYPE:AUTHOR_PREVIEW</value>
+				<value>UNINDEXED:TYPE:WORKFLOW</value>
+				<value>UNINDEXED:TYPE:WORKFLOW_PREVIEW</value>
+				<value>UNINDEXED:TYPE:AUTHOR_WORKFLOW</value>
+				<value>UNINDEXED:TYPE:AUTHOR_WORKFLOW_PREVIEW</value>
+				<value>ASYNCHRONOUS:NAME:avmAsynchronousTest</value>
+				<value>SYNCHRONOUS:NAME:.*</value>
+			</list>
+		</property>
     </bean>
     
     <!-- The AVMSyncService -->
@@ -1310,6 +1327,7 @@
         </property>
     </bean>
 
+	<!--
     <bean id="LinkValidationService" 
           class="org.springframework.aop.framework.ProxyFactoryBean"
           lazy-init="true">
@@ -1328,5 +1346,6 @@
             </list>
         </property>
     </bean>
+		 -->
 
 </beans>

source/java/org/alfresco/repo/avm/AVMServiceTest.java

@@ -37,6 +37,10 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 
+import javax.transaction.NotSupportedException;
+import javax.transaction.SystemException;
+import javax.transaction.UserTransaction;
+
 import org.alfresco.config.JNDIConstants;
 import org.alfresco.model.ContentModel;
 import org.alfresco.model.WCMModel;
@@ -49,6 +53,9 @@ import org.alfresco.repo.avm.actions.SimpleAVMSubmitAction;
 import org.alfresco.repo.avm.util.BulkLoader;
 import org.alfresco.repo.content.MimetypeMap;
 import org.alfresco.repo.domain.PropertyValue;
+import org.alfresco.repo.search.IndexMode;
+import org.alfresco.repo.search.Indexer;
+import org.alfresco.repo.search.impl.lucene.AVMLuceneIndexer;
 import org.alfresco.repo.search.impl.lucene.LuceneQueryParser;
 import org.alfresco.repo.search.impl.lucene.analysis.NumericEncoder;
 import org.alfresco.repo.transaction.RetryingTransactionHelper;
@@ -94,6 +101,86 @@ import org.alfresco.util.Pair;
  */
 public class AVMServiceTest extends AVMServiceTestBase
 {
+    /**
+     * Test async indexing.
+     * @throws Exception
+     */
+    public void testAsyncIndex() throws Exception
+    {
+        // Make sure the slate is clean ...
+        UserTransaction tx = fTransactionService.getUserTransaction();
+        tx.begin();
+        if(fService.getStore("avmAsynchronousTest") != null)
+        {
+            fService.purgeStore("avmAsynchronousTest");
+        }
+        StoreRef storeRef = AVMNodeConverter.ToStoreRef("avmAsynchronousTest");
+        Indexer indexer = fIndexerAndSearcher.getIndexer(storeRef);
+        if(indexer instanceof AVMLuceneIndexer)
+        {
+            AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer)indexer;
+            avmIndexer.deleteIndex("avmAsynchronousTest", IndexMode.SYNCHRONOUS);
+        }
+        tx.commit();
+      
+        // TODO: Suspend and resume indexing in case we are really unlucky and hit an index before we expect it.
+        
+        SearchService searchService = fIndexerAndSearcher.getSearcher(storeRef, true);
+        ResultSet results;
+        
+        results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
+        assertEquals(0, results.length());
+        results.close();
+        
+        fService.createStore("avmAsynchronousTest");
+        fService.createSnapshot("avmAsynchronousTest", null, null);
+        
+        results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
+        assertEquals(1, results.length());
+        results.close();
+        
+        fService.createDirectory("avmAsynchronousTest:/", "a");
+        fService.createDirectory("avmAsynchronousTest:/a", "b");
+        fService.createDirectory("avmAsynchronousTest:/a/b", "c");
+        fService.createSnapshot("avmAsynchronousTest", null, null);
+        
+        results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
+        assertEquals(1, results.length());
+        results.close();
+        
+        Thread.sleep(120000);
+        
+        results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
+        assertEquals(4, results.length());
+        results.close();
+        
+        fService.purgeStore("avmAsynchronousTest");
+        
+        results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
+        assertEquals(0, results.length());
+        results.close();
+        
+        fService.createStore("avmAsynchronousTest");
+        fService.createSnapshot("avmAsynchronousTest", null, null);
+        fService.createDirectory("avmAsynchronousTest:/", "a");
+        fService.createDirectory("avmAsynchronousTest:/a", "b");
+        fService.createDirectory("avmAsynchronousTest:/a/b", "c");
+        fService.createSnapshot("avmAsynchronousTest", null, null);
+        fService.purgeStore("avmAsynchronousTest");
+        fService.createStore("avmAsynchronousTest");
+        fService.createSnapshot("avmAsynchronousTest", null, null);
+        fService.createDirectory("avmAsynchronousTest:/", "a");
+        fService.createDirectory("avmAsynchronousTest:/a", "b");
+        fService.createDirectory("avmAsynchronousTest:/a/b", "c");
+        fService.createSnapshot("avmAsynchronousTest", null, null);
+        
+        Thread.sleep(120000);
+        
+        results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
+        assertEquals(4, results.length());
+        results.close();
+    }
+    
     public void testForceCopyDeleted()
     {
         try
@@ -125,7 +212,6 @@ public class AVMServiceTest extends AVMServiceTestBase
             runQueriesAgainstBasicTree("main");
             fService.createFile("main:/a", "Xander");
             fService.createSnapshot("layer", null, null);
-            runQueriesAgainstBasicTree("main");
             assertEquals(2, fService.lookup(2, "layer:/a").getIndirectionVersion());
             assertEquals(fService.lookup(2, "main:/a/Xander").getId(), fService.lookup(2, "layer:/a/Xander").getId());
             assertNull(fService.lookup(1, "layer:/a/Xander"));

source/java/org/alfresco/repo/avm/AVMServiceTestBase.java

@@ -45,6 +45,7 @@ import org.alfresco.service.cmr.search.ResultSet;
 import org.alfresco.service.cmr.search.ResultSetRow;
 import org.alfresco.service.cmr.search.SearchService;
 import org.alfresco.service.cmr.security.AuthenticationService;
+import org.alfresco.service.transaction.TransactionService;
 import org.springframework.context.support.FileSystemXmlApplicationContext;
 
 import junit.framework.TestCase;
@@ -79,6 +80,8 @@ public class AVMServiceTestBase extends TestCase
      */
     private long fStartTime;
 
+    protected TransactionService fTransactionService;
+
     protected static IndexerAndSearcher fIndexerAndSearcher;
     
     /**
@@ -96,6 +99,7 @@ public class AVMServiceTestBase extends TestCase
             fReaper = (OrphanReaper)fContext.getBean("orphanReaper");
             fSyncService = (AVMSyncService)fContext.getBean("AVMSyncService");
             fIndexerAndSearcher = (IndexerAndSearcher)fContext.getBean("indexerAndSearcherFactory");
+            fTransactionService = (TransactionService)fContext.getBean("transactionComponent");
             AuthenticationService authService = (AuthenticationService)fContext.getBean("AuthenticationService");
             authService.authenticate("admin", "admin".toCharArray());
             CreateStoreTxnListener cstl = (CreateStoreTxnListener)fContext.getBean("createStoreTxnListener");

source/java/org/alfresco/repo/search/AVMSnapShotTriggeredIndexingMethodInterceptor.java

@@ -24,46 +24,109 @@
  */
 package org.alfresco.repo.search;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import org.alfresco.error.AlfrescoRuntimeException;
 import org.alfresco.repo.avm.AVMNodeConverter;
+import org.alfresco.repo.domain.PropertyValue;
 import org.alfresco.repo.search.impl.lucene.AVMLuceneIndexer;
 import org.alfresco.service.cmr.avm.AVMService;
+import org.alfresco.service.cmr.avm.AVMStoreDescriptor;
 import org.alfresco.service.cmr.repository.StoreRef;
+import org.alfresco.service.namespace.QName;
 import org.aopalliance.intercept.MethodInterceptor;
 import org.aopalliance.intercept.MethodInvocation;
 
 /**
  * Method interceptor for atomic indexing of AVM entries
  * 
+ * The proeprties can defined how stores are indexed based on type (as set by Alfresco the Web site management UI)
+ * or based on the name of the store.
+ * 
+ * Creates and deletes are indexed synchronously.
+ * 
+ * Updates may be asynchronous, synchronous or ignored by the index.
+ * 
  * @author andyh
  */
 public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInterceptor
 {
+    // Copy of store properties used to tag avm stores (a store propertry)
+
+    public final static QName PROP_SANDBOX_STAGING_MAIN = QName.createQName(null, ".sandbox.staging.main");
+
+    public final static QName PROP_SANDBOX_STAGING_PREVIEW = QName.createQName(null, ".sandbox.staging.preview");
+
+    public final static QName PROP_SANDBOX_AUTHOR_MAIN = QName.createQName(null, ".sandbox.author.main");
+
+    public final static QName PROP_SANDBOX_AUTHOR_PREVIEW = QName.createQName(null, ".sandbox.author.preview");
+
+    public final static QName PROP_SANDBOX_WORKFLOW_MAIN = QName.createQName(null, ".sandbox.workflow.main");
+
+    public final static QName PROP_SANDBOX_WORKFLOW_PREVIEW = QName.createQName(null, ".sandbox.workflow.preview");
+
+    public final static QName PROP_SANDBOX_AUTHOR_WORKFLOW_MAIN = QName.createQName(null,
+            ".sandbox.author.workflow.main");
+
+    public final static QName PROP_SANDBOX_AUTHOR_WORKFLOW_PREVIEW = QName.createQName(null,
+            ".sandbox.author.workflow.preview");
+
     private AVMService avmService;
 
     private IndexerAndSearcher indexerAndSearcher;
 
     private boolean enableIndexing = true;
 
+    private IndexMode defaultMode = IndexMode.ASYNCHRONOUS;
+
+    private Map<String, IndexMode> modeCache = new HashMap<String, IndexMode>();
+
+    private List<IndexingDefinition> indexingDefinitions = new ArrayList<IndexingDefinition>();
+
     public Object invoke(MethodInvocation mi) throws Throwable
     {
         if (enableIndexing)
         {
             if (mi.getMethod().getName().equals("createSnapshot"))
             {
-                String store = (String) mi.getArguments()[0];
+                // May cause any number of other stores to do snap shot under the covers via layering or do nothing
-                int before = avmService.getLatestSnapshotID(store);
+                // So we have to watch what actually changes 
-                Object returnValue = mi.proceed();
+
-                int after = avmService.getLatestSnapshotID(store);
+                List<AVMStoreDescriptor> stores = avmService.getStores();
-                StoreRef storeRef = AVMNodeConverter.ToStoreRef(store);
+                Map<String, Integer> initialStates = new HashMap<String, Integer>(stores.size(), 1.0f);
-                Indexer indexer = indexerAndSearcher.getIndexer(storeRef);
+                for (AVMStoreDescriptor desc : stores)
-                if (indexer instanceof AVMLuceneIndexer)
                 {
-                    AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
+                    String store = desc.getName();
-                    avmIndexer.index(store, before, after);
+                    Integer state = Integer.valueOf(avmService.getLatestSnapshotID(store));
+                    initialStates.put(store, state);
+                }
+
+                Object returnValue = mi.proceed();
+
+                // Index any stores that have moved on
+                for (AVMStoreDescriptor desc : stores)
+                {
+                    String store = desc.getName();
+                    int after = avmService.getLatestSnapshotID(store);
+                    int before = initialStates.get(store).intValue();
+                    if (after > before)
+                    {
+
+                        StoreRef storeRef = AVMNodeConverter.ToStoreRef(store);
+                        Indexer indexer = indexerAndSearcher.getIndexer(storeRef);
+                        if (indexer instanceof AVMLuceneIndexer)
+                        {
+                            AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
+                            avmIndexer.index(store, before, after, getIndexMode(store));
+                        }
+                    }
                 }
                 return returnValue;
             }
-            // TODO: Purge store
             else if (mi.getMethod().getName().equals("purgeStore"))
             {
                 String store = (String) mi.getArguments()[0];
@@ -73,7 +136,7 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte
                 if (indexer instanceof AVMLuceneIndexer)
                 {
                     AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
-                    avmIndexer.deleteIndex(store);
+                    avmIndexer.deleteIndex(store, IndexMode.SYNCHRONOUS);
                 }
                 return returnValue;
             }
@@ -86,7 +149,7 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte
                 if (indexer instanceof AVMLuceneIndexer)
                 {
                     AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
-                    avmIndexer.createIndex(store);
+                    avmIndexer.createIndex(store, IndexMode.SYNCHRONOUS);
                 }
                 return returnValue;
             }
@@ -104,15 +167,15 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte
                 if (indexer instanceof AVMLuceneIndexer)
                 {
                     AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
-                    avmIndexer.deleteIndex(from);
+                    avmIndexer.deleteIndex(from, IndexMode.SYNCHRONOUS);
                 }
 
                 indexer = indexerAndSearcher.getIndexer(toRef);
                 if (indexer instanceof AVMLuceneIndexer)
                 {
                     AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
-                    avmIndexer.createIndex(to);
+                    avmIndexer.createIndex(to, IndexMode.SYNCHRONOUS);
-                    avmIndexer.index(to, 0, after);
+                    avmIndexer.index(to, 0, after, getIndexMode(to));
                 }
 
                 return returnValue;
@@ -157,7 +220,147 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte
     {
         this.enableIndexing = enableIndexing;
     }
-    
-    
 
+    /**
+     * Set the index modes.... Strings of the form ... (ASYNCHRONOUS | SYNCHRONOUS | UNINDEXED):(NAME | TYPE):regexp
+     * 
+     * @param definitions
+     */
+    public void setIndexingDefinitions(List<String> definitions)
+    {
+        indexingDefinitions.clear();
+        for (String def : definitions)
+        {
+            IndexingDefinition id = new IndexingDefinition(def);
+            indexingDefinitions.add(id);
+        }
+    }
+
+    /**
+     * Set the default index mode = used when there are no matches
+     * 
+     * @param defaultMode
+     */
+    public void setDefaultMode(IndexMode defaultMode)
+    {
+        this.defaultMode = defaultMode;
+    }
+
+    private synchronized IndexMode getIndexMode(String store)
+    {
+        IndexMode mode = modeCache.get(store);
+        if (mode == null)
+        {
+            for (IndexingDefinition def : indexingDefinitions)
+            {
+                if (def.definitionType == DefinitionType.NAME)
+                {
+                    if (def.pattern.matcher(store).matches())
+                    {
+                        mode = def.indexMode;
+                        modeCache.put(store, mode);
+                        break;
+                    }
+                }
+                else
+                {
+                    String storeType = getStoreType(store).toString();
+                    if (def.pattern.matcher(storeType).matches())
+                    {
+                        mode = def.indexMode;
+                        modeCache.put(store, mode);
+                        break;
+                    }
+
+                }
+            }
+        }
+        // No definition
+        if (mode == null)
+        {
+            mode = defaultMode;
+            modeCache.put(store, mode);
+        }
+        return mode;
+    }
+
+    private class IndexingDefinition
+    {
+        IndexMode indexMode;
+
+        DefinitionType definitionType;
+
+        Pattern pattern;
+
+        IndexingDefinition(String definition)
+        {
+            String[] split = definition.split(":", 3);
+            if (split.length != 3)
+            {
+                throw new AlfrescoRuntimeException(
+                        "Invalid index defintion. Must be of of the form IndexMode:DefinitionType:regular expression");
+            }
+            indexMode = IndexMode.valueOf(split[0].toUpperCase());
+            definitionType = DefinitionType.valueOf(split[1].toUpperCase());
+            pattern = Pattern.compile(split[2]);
+        }
+    }
+
+    private StoreType getStoreType(String name)
+    {
+        if (avmService.getStore(name) != null)
+        {
+            Map<QName, PropertyValue> storeProperties = avmService.getStoreProperties(name);
+            if (storeProperties.containsKey(PROP_SANDBOX_STAGING_MAIN))
+            {
+                return StoreType.STAGING;
+            }
+            else if (storeProperties.containsKey(PROP_SANDBOX_STAGING_PREVIEW))
+            {
+                return StoreType.STAGING_PREVIEW;
+            }
+            else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_MAIN))
+            {
+                return StoreType.AUTHOR;
+            }
+            else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_PREVIEW))
+            {
+                return StoreType.AUTHOR_PREVIEW;
+            }
+            else if (storeProperties.containsKey(PROP_SANDBOX_WORKFLOW_MAIN))
+            {
+                return StoreType.WORKFLOW;
+            }
+            else if (storeProperties.containsKey(PROP_SANDBOX_WORKFLOW_PREVIEW))
+            {
+                return StoreType.WORKFLOW_PREVIEW;
+            }
+            else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_WORKFLOW_MAIN))
+            {
+                return StoreType.AUTHOR_WORKFLOW;
+            }
+            else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_WORKFLOW_PREVIEW))
+            {
+                return StoreType.AUTHOR_WORKFLOW_PREVIEW;
+            }
+            else
+            {
+                return StoreType.UNKNOWN;
+            }
+        }
+        else
+        {
+            return StoreType.UNKNOWN;
+        }
+    }
+
+    private enum DefinitionType
+    {
+        NAME, TYPE;
+    }
+
+    private enum StoreType
+    {
+        STAGING, STAGING_PREVIEW, AUTHOR, AUTHOR_PREVIEW, WORKFLOW, WORKFLOW_PREVIEW, AUTHOR_WORKFLOW, AUTHOR_WORKFLOW_PREVIEW, UNKNOWN;
+    }
 }

source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexer.java

@@ -24,13 +24,16 @@
  */
 package org.alfresco.repo.search.impl.lucene;
 
+import org.alfresco.repo.search.BackgroundIndexerAware;
+import org.alfresco.repo.search.IndexMode;
+
 /**
  * AVM specific indxer support
  * 
  * @author andyh
  *
  */
-public interface AVMLuceneIndexer extends LuceneIndexer
+public interface AVMLuceneIndexer extends LuceneIndexer, BackgroundIndexerAware
 {
     /**
      * Index a specified change to a store between two snapshots 
@@ -38,21 +41,24 @@ public interface AVMLuceneIndexer extends LuceneIndexer
      * @param store - the name of the store
      * @param srcVersion - the id of the snapshot before the changeset
      * @param dstVersion - the id of the snapshot created by the change set
+     * @param mode 
      */
-    public void index(String store, int srcVersion, int dstVersion);
+    public void index(String store, int srcVersion, int dstVersion, IndexMode mode);
     
     /**
      * Delete the index for the specified store.
      * 
      * @param store
+     * @param mode 
      */
-    public void deleteIndex(String store);
+    public void deleteIndex(String store, IndexMode mode);
     
     /**
      * Create an index for the specified store.
      * This makes sure that the root node for the store is indexed correctly.
      * 
      * @param store
+     * @param mode
      */
-    public void createIndex(String store);
+    public void createIndex(String store, IndexMode mode);
 }

source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerAndSearcherFactory.java

@@ -30,6 +30,8 @@ import java.util.List;
 import org.alfresco.repo.avm.AVMNodeConverter;
 import org.alfresco.repo.content.ContentStore;
 import org.alfresco.repo.search.SearcherException;
+import org.alfresco.repo.search.SupportsBackgroundIndexing;
+import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer;
 import org.alfresco.service.cmr.avm.AVMService;
 import org.alfresco.service.cmr.avm.AVMStoreDescriptor;
 import org.alfresco.service.cmr.avmsync.AVMSyncService;
@@ -47,7 +49,7 @@ import org.apache.commons.logging.LogFactory;
  * @author andyh
  *
  */
-public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAndSearcherFactory
+public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAndSearcherFactory implements SupportsBackgroundIndexing
 {
     private static Log s_logger = LogFactory.getLog(AVMLuceneIndexerAndSearcherFactory.class);
     
@@ -65,6 +67,8 @@ public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAnd
 
     private ContentStore contentStore;
 
+    private FullTextSearchIndexer fullTextSearchIndexer;
+
     public AVMLuceneIndexerAndSearcherFactory()
     {
         //s_logger.error("Creating AVMLuceneIndexerAndSearcherFactory");
@@ -145,6 +149,7 @@ public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAnd
         indexer.setAvmService(avmService);
         indexer.setAvmSyncService(avmSyncService);
         indexer.setContentStore(contentStore);
+        indexer.setFullTextSearchIndexer(fullTextSearchIndexer);
         return indexer;
     }
 
@@ -174,4 +179,13 @@ public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAnd
         return searcher;
     }
 
+    /** 
+     * Register the full text searcher (done by the seracher bean to break cyclic bean defs) 
+     * 
+     */
+    public void setFullTextSearchIndexer(FullTextSearchIndexer fullTextSearchIndexer)
+    {
+        this.fullTextSearchIndexer = fullTextSearchIndexer;
+    }
+
 }

source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerImpl.java

@@ -51,6 +51,10 @@ import org.alfresco.repo.content.ContentStore;
 import org.alfresco.repo.content.MimetypeMap;
 import org.alfresco.repo.content.transform.ContentTransformer;
 import org.alfresco.repo.domain.PropertyValue;
+import org.alfresco.repo.search.IndexMode;
+import org.alfresco.repo.search.impl.lucene.analysis.NumericEncoder;
+import org.alfresco.repo.search.impl.lucene.fts.FTSIndexerAware;
+import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer;
 import org.alfresco.repo.transaction.AlfrescoTransactionSupport;
 import org.alfresco.service.cmr.avm.AVMException;
 import org.alfresco.service.cmr.avm.AVMNodeDescriptor;
@@ -72,11 +76,16 @@ import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
 import org.alfresco.service.cmr.repository.datatype.TypeConversionException;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.util.EqualsHelper;
+import org.alfresco.util.GUID;
 import org.alfresco.util.ISO9075;
 import org.alfresco.util.Pair;
 import org.apache.log4j.Logger;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Searcher;
 
 /**
  * Update the index after a snap shot to an AVM store. (Revert is dealt with as a new snap shot is created)
@@ -85,6 +94,10 @@ import org.apache.lucene.document.Field;
  */
 public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> implements AVMLuceneIndexer
 {
+    private static String SNAP_SHOT_ID = "SnapShotId_";
+
+    private static String SNAP_SHOT_STORE = "SnapShotStore";
+
     static Logger s_logger = Logger.getLogger(AVMLuceneIndexerImpl.class);
 
     private AVMService avmService;
@@ -98,8 +111,19 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
 
     private Set<String> indexedPaths = new HashSet<String>();
 
+    private FTSIndexerAware callBack;
+
+    private FullTextSearchIndexer fullTextSearchIndexer;
+
+    private int remainingCount;
+    
+    private int startVersion = -1;
+    
+    private int endVersion = -1;
+
     /**
      * Set the AVM Service
+     * 
      * @param avmService
      */
     public void setAvmService(AVMService avmService)
@@ -109,6 +133,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
 
     /**
      * Set the AVM sync service
+     * 
      * @param avmSyncService
      */
     public void setAvmSyncService(AVMSyncService avmSyncService)
@@ -118,6 +143,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
 
     /**
      * Set the content service
+     * 
      * @param contentStore
      */
     public void setContentStore(ContentStore contentStore)
@@ -162,10 +188,55 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
      * @param srcVersion
      * @param dstVersion
      */
-    public void index(String store, int srcVersion, int dstVersion)
+    public void index(String store, int srcVersion, int dstVersion, IndexMode mode)
     {
         checkAbleToDoWork(IndexUpdateStatus.SYNCRONOUS);
+        try
+        {
+            switch (mode)
+            {
+            case ASYNCHRONOUS:
+                asynchronousIndex(store, srcVersion, dstVersion);
+                break;
+            case SYNCHRONOUS:
+                synchronousIndex(store, srcVersion, dstVersion);
+                break;
+            case UNINDEXED:
+                // nothing to do
+                break;
+            }
+        }
+        catch (LuceneIndexException e)
+        {
+            setRollbackOnly();
+            throw new LuceneIndexException("snapshot index failed", e);
+        }
+    }
 
+    public void asynchronousIndex(String store, int srcVersion, int dstVersion)
+    {
+        if(s_logger.isDebugEnabled())
+        {
+            s_logger.debug("Async index for "+store + " from " + srcVersion + " to " + dstVersion);
+        }
+        index("\u0000BG:STORE:" + store + ":" + srcVersion + ":" + dstVersion + ":" + GUID.generate());
+        fullTextSearchIndexer.requiresIndex(AVMNodeConverter.ToStoreRef(store));
+    }
+
+    public void synchronousIndex(String store, int srcVersion, int dstVersion)
+    {
+        if(startVersion == -1)
+        {
+            startVersion = srcVersion;
+        }
+       
+        endVersion = dstVersion;
+            
+        
+        if(s_logger.isDebugEnabled())
+        {
+            s_logger.debug("Sync index for "+store + " from " + srcVersion + " to "+dstVersion);
+        }
         String path = store + ":/";
         List<AVMDifference> changeList = avmSyncService.compare(srcVersion, path, dstVersion, path, null);
         for (AVMDifference difference : changeList)
@@ -221,6 +292,8 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
 
             }
         }
+        // record the snap shotid
+        reindex(SNAP_SHOT_ID + store, false);
     }
 
     /*
@@ -232,14 +305,14 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
         String store = splitPath[0];
         String pathInStore = splitPath[1];
         SimplePath simplePath = new SimplePath(pathInStore);
-        
+
         StringBuilder pathBuilder = new StringBuilder();
         pathBuilder.append(store).append(":/");
         reindex(pathBuilder.toString(), false);
         boolean requiresSep = false;
         for (int i = 0; i < simplePath.size() - 1; i++)
         {
-            if(requiresSep)
+            if (requiresSep)
             {
                 pathBuilder.append("/");
             }
@@ -270,25 +343,64 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
     protected List<Document> createDocuments(String stringNodeRef, boolean isNew, boolean indexAllProperties,
             boolean includeDirectoryDocuments)
     {
-        AVMNodeDescriptor desc = avmService.lookup(-1, stringNodeRef);
         List<Document> docs = new ArrayList<Document>();
-        if(desc == null)
+        if (stringNodeRef.startsWith("\u0000"))
+        {
+            Document idoc = new Document();
+            idoc.add(new Field("ID", stringNodeRef, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+            docs.add(idoc);
+            return docs;
+        }
+        else if (stringNodeRef.startsWith(SNAP_SHOT_ID))
+        {
+            Document sdoc = new Document();
+            sdoc.add(new Field("ID", stringNodeRef, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+            String storeName = stringNodeRef.substring(SNAP_SHOT_ID.length());
+            sdoc.add(new Field(SNAP_SHOT_ID, NumericEncoder.encode(avmService.getLatestSnapshotID(storeName)),
+                    Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+            sdoc.add(new Field(SNAP_SHOT_STORE, NumericEncoder.encode(avmService.getLatestSnapshotID(storeName)),
+                    Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+            docs.add(sdoc);
+            return docs;
+        }
+
+        AVMNodeDescriptor desc = avmService.lookup(endVersion, stringNodeRef);
+        if (desc == null)
         {
             return docs;
         }
         if (desc.isLayeredDirectory() || desc.isLayeredFile())
         {
             return docs;
-        }   
+        }
-        
+
-        List<Pair<Integer, String>> paths = avmService.getHeadPaths(desc);
+        List<Pair<Integer, String>> allPaths = avmService.getPaths(desc);
+        List<Pair<Integer, String>> paths = new ArrayList<Pair<Integer, String>> ();
+        for(Pair<Integer, String> pair: allPaths)
+        {
+            if(pair.getFirst().intValue() == endVersion)
+            {
+                paths.add(pair);
+            }
+        }
         if(paths.size() == 0)
+        {
+            for(Pair<Integer, String> pair: allPaths)
+            {
+                if(pair.getFirst().intValue() == -1)
+                {
+                    paths.add(pair);
+                }
+            }
+        }
+            
+        if (paths.size() == 0)
         {
             return docs;
         }
         for (Pair<Integer, String> path : paths)
         {
-            if(indexedPaths.contains(path.getSecond()))
+            if (indexedPaths.contains(path.getSecond()))
             {
                 return docs;
             }
@@ -297,37 +409,35 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
         {
             indexedPaths.add(path.getSecond());
         }
-            
+
         AVMNode node = AVMDAOs.Instance().fAVMNodeDAO.getByID(desc.getId());
-       
 
         if (paths.size() > 0)
         {
             if (desc != null)
             {
 
-                NodeRef nodeRef = AVMNodeConverter.ToNodeRef(-1, stringNodeRef);
+                NodeRef nodeRef = AVMNodeConverter.ToNodeRef(endVersion, stringNodeRef);
 
                 Document xdoc = new Document();
-                xdoc.add(new Field("ID", nodeRef.toString(), Field.Store.YES,
+                xdoc.add(new Field("ID", nodeRef.toString(), Field.Store.YES, Field.Index.UN_TOKENIZED,
-                        Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                        Field.TermVector.NO));
                 for (Pair<Integer, String> path : paths)
                 {
 
                     String[] splitPath = splitPath(path.getSecond());
                     @SuppressWarnings("unused")
                     String pathInStore = splitPath[1];
-                    xdoc.add(new Field("ID", path.getSecond(),
+                    xdoc.add(new Field("ID", path.getSecond(), Field.Store.YES, Field.Index.UN_TOKENIZED,
-                            Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                            Field.TermVector.NO));
                 }
-              
 
                 xdoc.add(new Field("TX", AlfrescoTransactionSupport.getTransactionId(), Field.Store.YES,
                         Field.Index.UN_TOKENIZED, Field.TermVector.NO));
 
                 boolean isAtomic = true;
 
-                Map<QName, Serializable> properties = getIndexableProperties(desc, node, nodeRef, -1, stringNodeRef);
+                Map<QName, Serializable> properties = getIndexableProperties(desc, node, nodeRef, endVersion, stringNodeRef);
                 for (QName propertyName : properties.keySet())
                 {
                     Serializable value = properties.get(propertyName);
@@ -357,9 +467,9 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
                         String store = splitPath[0];
                         String pathInStore = splitPath[1];
                         SimplePath simplePath = new SimplePath(pathInStore);
-                        
+
                         StringBuilder xpathBuilder = new StringBuilder();
-                        for(int i = 0; i < simplePath.size(); i++)
+                        for (int i = 0; i < simplePath.size(); i++)
                         {
                             xpathBuilder.append("/{}").append(simplePath.get(i));
                         }
@@ -379,7 +489,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
                         boolean requiresSep = false;
                         for (int i = 0; i < simplePath.size() - 1; i++)
                         {
-                            if(requiresSep)
+                            if (requiresSep)
                             {
                                 pathBuilder.append("/");
                             }
@@ -393,8 +503,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
 
                         qNameBuffer.append(ISO9075.getXPathName(QName.createQName("", simplePath
                                 .get(simplePath.size() - 1))));
-                        xdoc.add(new Field("PARENT",
+                        xdoc.add(new Field("PARENT", ancestors.get(ancestors.size() - 1), Field.Store.YES,
-                                ancestors.get(ancestors.size() - 1), Field.Store.YES,
                                 Field.Index.UN_TOKENIZED, Field.TermVector.NO));
                         // TODO: Categories and LINKASPECT
 
@@ -407,20 +516,19 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
                                 Document directoryEntry = new Document();
                                 directoryEntry.add(new Field("ID", nodeRef.toString(), Field.Store.YES,
                                         Field.Index.UN_TOKENIZED, Field.TermVector.NO));
-                               
+
-                                directoryEntry.add(new Field("ID", path.getSecond(),
+                                directoryEntry.add(new Field("ID", path.getSecond(), Field.Store.YES,
-                                            Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                                        Field.Index.UN_TOKENIZED, Field.TermVector.NO));
-                                
+
-                                
+                                directoryEntry.add(new Field("PATH", xpath, Field.Store.YES, Field.Index.TOKENIZED,
-                                directoryEntry.add(new Field("PATH", xpath, Field.Store.YES,
+                                        Field.TermVector.NO));
-                                        Field.Index.TOKENIZED, Field.TermVector.NO));
 
                                 // Find all parent nodes.
 
                                 for (String toAdd : ancestors)
                                 {
-                                    directoryEntry.add(new Field("ANCESTOR", toAdd,
+                                    directoryEntry.add(new Field("ANCESTOR", toAdd, Field.Store.NO,
-                                            Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                                            Field.Index.UN_TOKENIZED, Field.TermVector.NO));
                                 }
                                 directoryEntry.add(new Field("ISCONTAINER", "T", Field.Store.YES,
                                         Field.Index.UN_TOKENIZED, Field.TermVector.NO));
@@ -456,7 +564,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
                     xdoc.add(new Field("TYPE", ISO9075.getXPathName(typeQName), Field.Store.YES,
                             Field.Index.UN_TOKENIZED, Field.TermVector.NO));
 
-                    for (QName classRef : avmService.getAspects(-1, stringNodeRef))
+                    for (QName classRef : avmService.getAspects(endVersion, stringNodeRef))
                     {
                         xdoc.add(new Field("ASPECT", ISO9075.getXPathName(classRef), Field.Store.YES,
                                 Field.Index.UN_TOKENIZED, Field.TermVector.NO));
@@ -534,11 +642,11 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
         result.put(ContentModel.PROP_STORE_IDENTIFIER, nodeRef.getStoreRef().getIdentifier());
         if (desc.isLayeredDirectory())
         {
-            result.put(WCMModel.PROP_AVM_DIR_INDIRECTION, AVMNodeConverter.ToNodeRef(-1, desc.getIndirection()));
+            result.put(WCMModel.PROP_AVM_DIR_INDIRECTION, AVMNodeConverter.ToNodeRef(endVersion, desc.getIndirection()));
         }
         if (desc.isLayeredFile())
         {
-            result.put(WCMModel.PROP_AVM_FILE_INDIRECTION, AVMNodeConverter.ToNodeRef(-1, desc.getIndirection()));
+            result.put(WCMModel.PROP_AVM_FILE_INDIRECTION, AVMNodeConverter.ToNodeRef(endVersion, desc.getIndirection()));
         }
         if (desc.isFile())
         {
@@ -873,13 +981,32 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
     @Override
     protected void doCommit() throws IOException
     {
+        if (indexUpdateStatus == IndexUpdateStatus.ASYNCHRONOUS)
+        {
+            setInfo(docs, getDeletions(), false);
+            // FTS does not trigger indexing request
+        }
+        else
+        {
+            setInfo(docs, getDeletions(), false);
+            // TODO: only register if required
+            fullTextSearchIndexer.requiresIndex(store);
+        }
+        if (callBack != null)
+        {
+            callBack.indexCompleted(store, remainingCount, null);
+        }
+
         setInfo(docs, deletions, false);
     }
 
     @Override
     protected void doRollBack() throws IOException
     {
-
+        if (callBack != null)
+        {
+            callBack.indexCompleted(store, 0, null);
+        }
     }
 
     @Override
@@ -1014,29 +1141,68 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
 
     }
 
-    public void deleteIndex(String store)
+    public void deleteIndex(String store, IndexMode mode)
     {
         checkAbleToDoWork(IndexUpdateStatus.SYNCRONOUS);
         try
         {
-            deleteAll();
+            switch (mode)
+            {
+            case ASYNCHRONOUS:
+                asyncronousDeleteIndex(store);
+                break;
+            case SYNCHRONOUS:
+                syncronousDeleteIndex(store);
+                break;
+            case UNINDEXED:
+                // nothing to do
+                break;
+            }
         }
         catch (LuceneIndexException e)
         {
             setRollbackOnly();
             throw new LuceneIndexException("Delete index failed", e);
         }
-
     }
 
-    public void createIndex(String store)
+    public void syncronousDeleteIndex(String store)
+    {
+
+        if(s_logger.isDebugEnabled())
+        {
+            s_logger.debug("Sync delete for "+store);
+        }
+        deleteAll();
+    }
+
+    public void asyncronousDeleteIndex(String store)
+    {
+        if(s_logger.isDebugEnabled())
+        {
+            s_logger.debug("Async delete for "+store);
+        }
+        index("\u0000BG:DELETE:" + store+":"+GUID.generate());
+        fullTextSearchIndexer.requiresIndex(AVMNodeConverter.ToStoreRef(store));
+    }
+
+    public void createIndex(String store, IndexMode mode)
     {
         checkAbleToDoWork(IndexUpdateStatus.SYNCRONOUS);
         try
         {
-            @SuppressWarnings("unused")
+            switch (mode)
-            AVMNodeDescriptor rootDesc = avmService.getStoreRoot(-1, store);
+            {
-            index(store+":/");
+            case ASYNCHRONOUS:
+                asyncronousCreateIndex(store);
+                break;
+            case SYNCHRONOUS:
+                syncronousCreateIndex(store);
+                break;
+            case UNINDEXED:
+                // nothing to do
+                break;
+            }
         }
         catch (LuceneIndexException e)
         {
@@ -1044,4 +1210,123 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
             throw new LuceneIndexException("Create index failed", e);
         }
     }
+
+    public void syncronousCreateIndex(String store)
+    {
+
+        if(s_logger.isDebugEnabled())
+        {
+            s_logger.debug("Sync create for "+store);
+        }
+        @SuppressWarnings("unused")
+        AVMNodeDescriptor rootDesc = avmService.getStoreRoot(-1, store);
+        index(store + ":/");
+
+    }
+
+    public void asyncronousCreateIndex(String store)
+    {
+        if(s_logger.isDebugEnabled())
+        {
+            s_logger.debug("Async create for "+store);
+        }
+        index("\u0000BG:CREATE:" + store+":"+GUID.generate());
+        fullTextSearchIndexer.requiresIndex(AVMNodeConverter.ToStoreRef(store));
+    }
+
+    public void registerCallBack(FTSIndexerAware callBack)
+    {
+        this.callBack = callBack;
+    }
+
+    public int updateFullTextSearch(int size)
+    {
+        checkAbleToDoWork(IndexUpdateStatus.ASYNCHRONOUS);
+
+        try
+        {
+            PrefixQuery query = new PrefixQuery(new Term("ID", "\u0000BG:"));
+
+            String action = null;
+
+            Searcher searcher = null;
+            try
+            {
+                searcher = getSearcher(null);
+                // commit on another thread - appears like there is no index ...try later
+                if (searcher == null)
+                {
+                    remainingCount = size;
+                    return 0;
+                }
+                Hits hits;
+                try
+                {
+                    hits = searcher.search(query);
+                }
+                catch (IOException e)
+                {
+                    throw new LuceneIndexException(
+                            "Failed to execute query to find content which needs updating in the index", e);
+                }
+
+                if (hits.length() > 0)
+                {
+                    Document doc = hits.doc(0);
+                    action = doc.getField("ID").stringValue();
+                    String[] split = action.split(":");
+                    if(split[1].equals("DELETE"))
+                    {
+                        deleteAll("\u0000BG:");
+                    }
+                    else if(split[1].equals("CREATE"))
+                    {
+                        syncronousCreateIndex(split[2]);
+                    }
+                    else if(split[1].equals("STORE"))
+                    {
+                        synchronousIndex(split[2], Integer.parseInt(split[3]), Integer.parseInt(split[4]));
+                    }
+                    deletions.add(action);
+                    remainingCount = hits.length() - 1;
+                    return 1;
+                }
+                else
+                {
+                    remainingCount = 0;
+                    return 0;
+                }
+
+            }
+            finally
+            {
+                if (searcher != null)
+                {
+                    try
+                    {
+                        searcher.close();
+                    }
+                    catch (IOException e)
+                    {
+                        throw new LuceneIndexException("Failed to close searcher", e);
+                    }
+                }
+            }
+        }
+        catch (IOException e)
+        {
+            setRollbackOnly();
+            throw new LuceneIndexException("Failed FTS update", e);
+        }
+        catch (LuceneIndexException e)
+        {
+            setRollbackOnly();
+            throw new LuceneIndexException("Failed FTS update", e);
+        }
+    }
+
+    public void setFullTextSearchIndexer(FullTextSearchIndexer fullTextSearchIndexer)
+    {
+        this.fullTextSearchIndexer = fullTextSearchIndexer;
+    }
 }

source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerImpl.java

@@ -63,15 +63,15 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
         /**
          * An index
          */
-        INDEX, 
+        INDEX,
         /**
          * A reindex
          */
-        REINDEX, 
+        REINDEX,
         /**
          * A delete
          */
-        DELETE, 
+        DELETE,
         /**
          * A cascaded reindex (ensures directory structre is ok)
          */
@@ -83,13 +83,13 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
         /**
          * Inde is unchanged
          */
-        UNMODIFIED, 
+        UNMODIFIED,
         /**
-         * Index is being changein in TX 
+         * Index is being changein in TX
          */
-        SYNCRONOUS, 
+        SYNCRONOUS,
         /**
-         * Index is eiong changed by a background upate 
+         * Index is eiong changed by a background upate
          */
         ASYNCHRONOUS;
     }
@@ -379,7 +379,8 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
 
     /**
      * Commit this index
-     * @throws LuceneIndexException 
+     * 
+     * @throws LuceneIndexException
      */
     public void commit() throws LuceneIndexException
     {
@@ -441,7 +442,7 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
      * serialisation against the index as would a data base transaction.
      * 
      * @return the tx state
-     * @throws LuceneIndexException 
+     * @throws LuceneIndexException
      */
     public int prepare() throws LuceneIndexException
     {
@@ -498,7 +499,8 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
 
     /**
      * Roll back the index changes (this just means they are never added)
-     * @throws LuceneIndexException 
+     * 
+     * @throws LuceneIndexException
      */
     public void rollback() throws LuceneIndexException
     {
@@ -819,6 +821,7 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
 
     /**
      * Are we deleting leaves only (not meta data)
+     * 
      * @return - deleting only nodes.
      */
     public boolean getDeleteOnlyNodes()
@@ -828,6 +831,7 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
 
     /**
      * Get the deletions
+     * 
      * @return - the ids to delete
      */
     public Set<String> getDeletions()
@@ -837,9 +841,18 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
 
     /**
      * Delete all entries from the index.
-     *
      */
     public void deleteAll()
+    {
+        deleteAll(null);
+    }
+
+    /**
+     * Delete all index entries which do not start with the goven prefix
+     * 
+     * @param prefix
+     */
+    public void deleteAll(String prefix)
     {
         IndexReader mainReader = null;
         try
@@ -851,7 +864,10 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
                 {
                     Document document = mainReader.document(doc);
                     String[] ids = document.getValues("ID");
-                    deletions.add(ids[ids.length - 1]);
+                    if ((prefix == null) || nonStartwWith(ids, prefix))
+                    {
+                        deletions.add(ids[ids.length - 1]);
+                    }
                 }
             }
 
@@ -877,4 +893,16 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
         }
     }
 
+    private boolean nonStartwWith(String[] values, String prefix)
+    {
+        for (String value : values)
+        {
+            if (value.startsWith(prefix))
+            {
+                return false;
+            }
+        }
+        return true;
+    }
+
 }