diff --git a/config/alfresco/public-services-context.xml b/config/alfresco/public-services-context.xml index 3409eb45ab..82279a54ae 100644 --- a/config/alfresco/public-services-context.xml +++ b/config/alfresco/public-services-context.xml @@ -892,6 +892,23 @@ true + + SYNCHRONOUS + + + + SYNCHRONOUS:TYPE:STAGING + UNINDEXED:TYPE:STAGING_PREVIEW + UNINDEXED:TYPE:AUTHOR + UNINDEXED:TYPE:AUTHOR_PREVIEW + UNINDEXED:TYPE:WORKFLOW + UNINDEXED:TYPE:WORKFLOW_PREVIEW + UNINDEXED:TYPE:AUTHOR_WORKFLOW + UNINDEXED:TYPE:AUTHOR_WORKFLOW_PREVIEW + ASYNCHRONOUS:NAME:avmAsynchronousTest + SYNCHRONOUS:NAME:.* + + @@ -1310,6 +1327,7 @@ + diff --git a/source/java/org/alfresco/repo/avm/AVMServiceTest.java b/source/java/org/alfresco/repo/avm/AVMServiceTest.java index 08a5e1fb92..738faeebb6 100644 --- a/source/java/org/alfresco/repo/avm/AVMServiceTest.java +++ b/source/java/org/alfresco/repo/avm/AVMServiceTest.java @@ -37,6 +37,10 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; +import javax.transaction.NotSupportedException; +import javax.transaction.SystemException; +import javax.transaction.UserTransaction; + import org.alfresco.config.JNDIConstants; import org.alfresco.model.ContentModel; import org.alfresco.model.WCMModel; @@ -49,6 +53,9 @@ import org.alfresco.repo.avm.actions.SimpleAVMSubmitAction; import org.alfresco.repo.avm.util.BulkLoader; import org.alfresco.repo.content.MimetypeMap; import org.alfresco.repo.domain.PropertyValue; +import org.alfresco.repo.search.IndexMode; +import org.alfresco.repo.search.Indexer; +import org.alfresco.repo.search.impl.lucene.AVMLuceneIndexer; import org.alfresco.repo.search.impl.lucene.LuceneQueryParser; import org.alfresco.repo.search.impl.lucene.analysis.NumericEncoder; import org.alfresco.repo.transaction.RetryingTransactionHelper; @@ -94,6 +101,86 @@ import org.alfresco.util.Pair; */ public class AVMServiceTest extends AVMServiceTestBase { + /** + * Test async indexing. + * @throws Exception + */ + public void testAsyncIndex() throws Exception + { + // Make sure the slate is clean ... + UserTransaction tx = fTransactionService.getUserTransaction(); + tx.begin(); + if(fService.getStore("avmAsynchronousTest") != null) + { + fService.purgeStore("avmAsynchronousTest"); + } + StoreRef storeRef = AVMNodeConverter.ToStoreRef("avmAsynchronousTest"); + Indexer indexer = fIndexerAndSearcher.getIndexer(storeRef); + if(indexer instanceof AVMLuceneIndexer) + { + AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer)indexer; + avmIndexer.deleteIndex("avmAsynchronousTest", IndexMode.SYNCHRONOUS); + } + tx.commit(); + + // TODO: Suspend and resume indexing in case we are really unlucky and hit an index before we expect it. + + SearchService searchService = fIndexerAndSearcher.getSearcher(storeRef, true); + ResultSet results; + + results = searchService.query(storeRef, "lucene", "PATH:\"//.\""); + assertEquals(0, results.length()); + results.close(); + + fService.createStore("avmAsynchronousTest"); + fService.createSnapshot("avmAsynchronousTest", null, null); + + results = searchService.query(storeRef, "lucene", "PATH:\"//.\""); + assertEquals(1, results.length()); + results.close(); + + fService.createDirectory("avmAsynchronousTest:/", "a"); + fService.createDirectory("avmAsynchronousTest:/a", "b"); + fService.createDirectory("avmAsynchronousTest:/a/b", "c"); + fService.createSnapshot("avmAsynchronousTest", null, null); + + results = searchService.query(storeRef, "lucene", "PATH:\"//.\""); + assertEquals(1, results.length()); + results.close(); + + Thread.sleep(120000); + + results = searchService.query(storeRef, "lucene", "PATH:\"//.\""); + assertEquals(4, results.length()); + results.close(); + + fService.purgeStore("avmAsynchronousTest"); + + results = searchService.query(storeRef, "lucene", "PATH:\"//.\""); + assertEquals(0, results.length()); + results.close(); + + fService.createStore("avmAsynchronousTest"); + fService.createSnapshot("avmAsynchronousTest", null, null); + fService.createDirectory("avmAsynchronousTest:/", "a"); + fService.createDirectory("avmAsynchronousTest:/a", "b"); + fService.createDirectory("avmAsynchronousTest:/a/b", "c"); + fService.createSnapshot("avmAsynchronousTest", null, null); + fService.purgeStore("avmAsynchronousTest"); + fService.createStore("avmAsynchronousTest"); + fService.createSnapshot("avmAsynchronousTest", null, null); + fService.createDirectory("avmAsynchronousTest:/", "a"); + fService.createDirectory("avmAsynchronousTest:/a", "b"); + fService.createDirectory("avmAsynchronousTest:/a/b", "c"); + fService.createSnapshot("avmAsynchronousTest", null, null); + + Thread.sleep(120000); + + results = searchService.query(storeRef, "lucene", "PATH:\"//.\""); + assertEquals(4, results.length()); + results.close(); + } + public void testForceCopyDeleted() { try @@ -125,7 +212,6 @@ public class AVMServiceTest extends AVMServiceTestBase runQueriesAgainstBasicTree("main"); fService.createFile("main:/a", "Xander"); fService.createSnapshot("layer", null, null); - runQueriesAgainstBasicTree("main"); assertEquals(2, fService.lookup(2, "layer:/a").getIndirectionVersion()); assertEquals(fService.lookup(2, "main:/a/Xander").getId(), fService.lookup(2, "layer:/a/Xander").getId()); assertNull(fService.lookup(1, "layer:/a/Xander")); diff --git a/source/java/org/alfresco/repo/avm/AVMServiceTestBase.java b/source/java/org/alfresco/repo/avm/AVMServiceTestBase.java index e49bc2a4eb..84a9e83416 100644 --- a/source/java/org/alfresco/repo/avm/AVMServiceTestBase.java +++ b/source/java/org/alfresco/repo/avm/AVMServiceTestBase.java @@ -45,6 +45,7 @@ import org.alfresco.service.cmr.search.ResultSet; import org.alfresco.service.cmr.search.ResultSetRow; import org.alfresco.service.cmr.search.SearchService; import org.alfresco.service.cmr.security.AuthenticationService; +import org.alfresco.service.transaction.TransactionService; import org.springframework.context.support.FileSystemXmlApplicationContext; import junit.framework.TestCase; @@ -79,6 +80,8 @@ public class AVMServiceTestBase extends TestCase */ private long fStartTime; + protected TransactionService fTransactionService; + protected static IndexerAndSearcher fIndexerAndSearcher; /** @@ -96,6 +99,7 @@ public class AVMServiceTestBase extends TestCase fReaper = (OrphanReaper)fContext.getBean("orphanReaper"); fSyncService = (AVMSyncService)fContext.getBean("AVMSyncService"); fIndexerAndSearcher = (IndexerAndSearcher)fContext.getBean("indexerAndSearcherFactory"); + fTransactionService = (TransactionService)fContext.getBean("transactionComponent"); AuthenticationService authService = (AuthenticationService)fContext.getBean("AuthenticationService"); authService.authenticate("admin", "admin".toCharArray()); CreateStoreTxnListener cstl = (CreateStoreTxnListener)fContext.getBean("createStoreTxnListener"); diff --git a/source/java/org/alfresco/repo/search/AVMSnapShotTriggeredIndexingMethodInterceptor.java b/source/java/org/alfresco/repo/search/AVMSnapShotTriggeredIndexingMethodInterceptor.java index f31da87319..2d7638ede8 100644 --- a/source/java/org/alfresco/repo/search/AVMSnapShotTriggeredIndexingMethodInterceptor.java +++ b/source/java/org/alfresco/repo/search/AVMSnapShotTriggeredIndexingMethodInterceptor.java @@ -24,46 +24,109 @@ */ package org.alfresco.repo.search; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.repo.avm.AVMNodeConverter; +import org.alfresco.repo.domain.PropertyValue; import org.alfresco.repo.search.impl.lucene.AVMLuceneIndexer; import org.alfresco.service.cmr.avm.AVMService; +import org.alfresco.service.cmr.avm.AVMStoreDescriptor; import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.namespace.QName; import org.aopalliance.intercept.MethodInterceptor; import org.aopalliance.intercept.MethodInvocation; /** * Method interceptor for atomic indexing of AVM entries * + * The proeprties can defined how stores are indexed based on type (as set by Alfresco the Web site management UI) + * or based on the name of the store. + * + * Creates and deletes are indexed synchronously. + * + * Updates may be asynchronous, synchronous or ignored by the index. + * * @author andyh */ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInterceptor { + // Copy of store properties used to tag avm stores (a store propertry) + + public final static QName PROP_SANDBOX_STAGING_MAIN = QName.createQName(null, ".sandbox.staging.main"); + + public final static QName PROP_SANDBOX_STAGING_PREVIEW = QName.createQName(null, ".sandbox.staging.preview"); + + public final static QName PROP_SANDBOX_AUTHOR_MAIN = QName.createQName(null, ".sandbox.author.main"); + + public final static QName PROP_SANDBOX_AUTHOR_PREVIEW = QName.createQName(null, ".sandbox.author.preview"); + + public final static QName PROP_SANDBOX_WORKFLOW_MAIN = QName.createQName(null, ".sandbox.workflow.main"); + + public final static QName PROP_SANDBOX_WORKFLOW_PREVIEW = QName.createQName(null, ".sandbox.workflow.preview"); + + public final static QName PROP_SANDBOX_AUTHOR_WORKFLOW_MAIN = QName.createQName(null, + ".sandbox.author.workflow.main"); + + public final static QName PROP_SANDBOX_AUTHOR_WORKFLOW_PREVIEW = QName.createQName(null, + ".sandbox.author.workflow.preview"); + private AVMService avmService; private IndexerAndSearcher indexerAndSearcher; private boolean enableIndexing = true; + private IndexMode defaultMode = IndexMode.ASYNCHRONOUS; + + private Map modeCache = new HashMap(); + + private List indexingDefinitions = new ArrayList(); + public Object invoke(MethodInvocation mi) throws Throwable { if (enableIndexing) { if (mi.getMethod().getName().equals("createSnapshot")) { - String store = (String) mi.getArguments()[0]; - int before = avmService.getLatestSnapshotID(store); - Object returnValue = mi.proceed(); - int after = avmService.getLatestSnapshotID(store); - StoreRef storeRef = AVMNodeConverter.ToStoreRef(store); - Indexer indexer = indexerAndSearcher.getIndexer(storeRef); - if (indexer instanceof AVMLuceneIndexer) + // May cause any number of other stores to do snap shot under the covers via layering or do nothing + // So we have to watch what actually changes + + List stores = avmService.getStores(); + Map initialStates = new HashMap(stores.size(), 1.0f); + for (AVMStoreDescriptor desc : stores) { - AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer; - avmIndexer.index(store, before, after); + String store = desc.getName(); + Integer state = Integer.valueOf(avmService.getLatestSnapshotID(store)); + initialStates.put(store, state); + } + + Object returnValue = mi.proceed(); + + // Index any stores that have moved on + for (AVMStoreDescriptor desc : stores) + { + String store = desc.getName(); + int after = avmService.getLatestSnapshotID(store); + int before = initialStates.get(store).intValue(); + if (after > before) + { + + StoreRef storeRef = AVMNodeConverter.ToStoreRef(store); + Indexer indexer = indexerAndSearcher.getIndexer(storeRef); + if (indexer instanceof AVMLuceneIndexer) + { + AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer; + avmIndexer.index(store, before, after, getIndexMode(store)); + } + } } return returnValue; } - // TODO: Purge store else if (mi.getMethod().getName().equals("purgeStore")) { String store = (String) mi.getArguments()[0]; @@ -73,7 +136,7 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte if (indexer instanceof AVMLuceneIndexer) { AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer; - avmIndexer.deleteIndex(store); + avmIndexer.deleteIndex(store, IndexMode.SYNCHRONOUS); } return returnValue; } @@ -86,7 +149,7 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte if (indexer instanceof AVMLuceneIndexer) { AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer; - avmIndexer.createIndex(store); + avmIndexer.createIndex(store, IndexMode.SYNCHRONOUS); } return returnValue; } @@ -104,15 +167,15 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte if (indexer instanceof AVMLuceneIndexer) { AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer; - avmIndexer.deleteIndex(from); + avmIndexer.deleteIndex(from, IndexMode.SYNCHRONOUS); } indexer = indexerAndSearcher.getIndexer(toRef); if (indexer instanceof AVMLuceneIndexer) { AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer; - avmIndexer.createIndex(to); - avmIndexer.index(to, 0, after); + avmIndexer.createIndex(to, IndexMode.SYNCHRONOUS); + avmIndexer.index(to, 0, after, getIndexMode(to)); } return returnValue; @@ -157,7 +220,147 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte { this.enableIndexing = enableIndexing; } - - + /** + * Set the index modes.... Strings of the form ... (ASYNCHRONOUS | SYNCHRONOUS | UNINDEXED):(NAME | TYPE):regexp + * + * @param definitions + */ + public void setIndexingDefinitions(List definitions) + { + indexingDefinitions.clear(); + for (String def : definitions) + { + IndexingDefinition id = new IndexingDefinition(def); + indexingDefinitions.add(id); + } + } + + /** + * Set the default index mode = used when there are no matches + * + * @param defaultMode + */ + public void setDefaultMode(IndexMode defaultMode) + { + this.defaultMode = defaultMode; + } + + private synchronized IndexMode getIndexMode(String store) + { + IndexMode mode = modeCache.get(store); + if (mode == null) + { + for (IndexingDefinition def : indexingDefinitions) + { + if (def.definitionType == DefinitionType.NAME) + { + if (def.pattern.matcher(store).matches()) + { + mode = def.indexMode; + modeCache.put(store, mode); + break; + } + } + else + { + String storeType = getStoreType(store).toString(); + if (def.pattern.matcher(storeType).matches()) + { + mode = def.indexMode; + modeCache.put(store, mode); + break; + } + + } + } + } + // No definition + if (mode == null) + { + mode = defaultMode; + modeCache.put(store, mode); + } + return mode; + } + + private class IndexingDefinition + { + IndexMode indexMode; + + DefinitionType definitionType; + + Pattern pattern; + + IndexingDefinition(String definition) + { + String[] split = definition.split(":", 3); + if (split.length != 3) + { + throw new AlfrescoRuntimeException( + "Invalid index defintion. Must be of of the form IndexMode:DefinitionType:regular expression"); + } + indexMode = IndexMode.valueOf(split[0].toUpperCase()); + definitionType = DefinitionType.valueOf(split[1].toUpperCase()); + pattern = Pattern.compile(split[2]); + } + } + + private StoreType getStoreType(String name) + { + if (avmService.getStore(name) != null) + { + Map storeProperties = avmService.getStoreProperties(name); + if (storeProperties.containsKey(PROP_SANDBOX_STAGING_MAIN)) + { + return StoreType.STAGING; + } + else if (storeProperties.containsKey(PROP_SANDBOX_STAGING_PREVIEW)) + { + return StoreType.STAGING_PREVIEW; + } + else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_MAIN)) + { + return StoreType.AUTHOR; + } + else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_PREVIEW)) + { + return StoreType.AUTHOR_PREVIEW; + } + else if (storeProperties.containsKey(PROP_SANDBOX_WORKFLOW_MAIN)) + { + return StoreType.WORKFLOW; + } + else if (storeProperties.containsKey(PROP_SANDBOX_WORKFLOW_PREVIEW)) + { + return StoreType.WORKFLOW_PREVIEW; + } + else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_WORKFLOW_MAIN)) + { + return StoreType.AUTHOR_WORKFLOW; + } + else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_WORKFLOW_PREVIEW)) + { + return StoreType.AUTHOR_WORKFLOW_PREVIEW; + } + else + { + return StoreType.UNKNOWN; + } + } + else + { + return StoreType.UNKNOWN; + } + } + + private enum DefinitionType + { + NAME, TYPE; + } + + private enum StoreType + { + STAGING, STAGING_PREVIEW, AUTHOR, AUTHOR_PREVIEW, WORKFLOW, WORKFLOW_PREVIEW, AUTHOR_WORKFLOW, AUTHOR_WORKFLOW_PREVIEW, UNKNOWN; + } } diff --git a/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexer.java b/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexer.java index b123deb44e..6991c5365c 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexer.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexer.java @@ -24,13 +24,16 @@ */ package org.alfresco.repo.search.impl.lucene; +import org.alfresco.repo.search.BackgroundIndexerAware; +import org.alfresco.repo.search.IndexMode; + /** * AVM specific indxer support * * @author andyh * */ -public interface AVMLuceneIndexer extends LuceneIndexer +public interface AVMLuceneIndexer extends LuceneIndexer, BackgroundIndexerAware { /** * Index a specified change to a store between two snapshots @@ -38,21 +41,24 @@ public interface AVMLuceneIndexer extends LuceneIndexer * @param store - the name of the store * @param srcVersion - the id of the snapshot before the changeset * @param dstVersion - the id of the snapshot created by the change set + * @param mode */ - public void index(String store, int srcVersion, int dstVersion); + public void index(String store, int srcVersion, int dstVersion, IndexMode mode); /** * Delete the index for the specified store. * * @param store + * @param mode */ - public void deleteIndex(String store); + public void deleteIndex(String store, IndexMode mode); /** * Create an index for the specified store. * This makes sure that the root node for the store is indexed correctly. * * @param store + * @param mode */ - public void createIndex(String store); + public void createIndex(String store, IndexMode mode); } diff --git a/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerAndSearcherFactory.java b/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerAndSearcherFactory.java index 069f528119..200cb3f9e9 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerAndSearcherFactory.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerAndSearcherFactory.java @@ -30,6 +30,8 @@ import java.util.List; import org.alfresco.repo.avm.AVMNodeConverter; import org.alfresco.repo.content.ContentStore; import org.alfresco.repo.search.SearcherException; +import org.alfresco.repo.search.SupportsBackgroundIndexing; +import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer; import org.alfresco.service.cmr.avm.AVMService; import org.alfresco.service.cmr.avm.AVMStoreDescriptor; import org.alfresco.service.cmr.avmsync.AVMSyncService; @@ -47,7 +49,7 @@ import org.apache.commons.logging.LogFactory; * @author andyh * */ -public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAndSearcherFactory +public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAndSearcherFactory implements SupportsBackgroundIndexing { private static Log s_logger = LogFactory.getLog(AVMLuceneIndexerAndSearcherFactory.class); @@ -65,6 +67,8 @@ public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAnd private ContentStore contentStore; + private FullTextSearchIndexer fullTextSearchIndexer; + public AVMLuceneIndexerAndSearcherFactory() { //s_logger.error("Creating AVMLuceneIndexerAndSearcherFactory"); @@ -145,6 +149,7 @@ public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAnd indexer.setAvmService(avmService); indexer.setAvmSyncService(avmSyncService); indexer.setContentStore(contentStore); + indexer.setFullTextSearchIndexer(fullTextSearchIndexer); return indexer; } @@ -174,4 +179,13 @@ public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAnd return searcher; } + /** + * Register the full text searcher (done by the seracher bean to break cyclic bean defs) + * + */ + public void setFullTextSearchIndexer(FullTextSearchIndexer fullTextSearchIndexer) + { + this.fullTextSearchIndexer = fullTextSearchIndexer; + } + } diff --git a/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerImpl.java b/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerImpl.java index 7228975e6e..b5b42d78f1 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerImpl.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerImpl.java @@ -51,6 +51,10 @@ import org.alfresco.repo.content.ContentStore; import org.alfresco.repo.content.MimetypeMap; import org.alfresco.repo.content.transform.ContentTransformer; import org.alfresco.repo.domain.PropertyValue; +import org.alfresco.repo.search.IndexMode; +import org.alfresco.repo.search.impl.lucene.analysis.NumericEncoder; +import org.alfresco.repo.search.impl.lucene.fts.FTSIndexerAware; +import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer; import org.alfresco.repo.transaction.AlfrescoTransactionSupport; import org.alfresco.service.cmr.avm.AVMException; import org.alfresco.service.cmr.avm.AVMNodeDescriptor; @@ -72,11 +76,16 @@ import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; import org.alfresco.service.cmr.repository.datatype.TypeConversionException; import org.alfresco.service.namespace.QName; import org.alfresco.util.EqualsHelper; +import org.alfresco.util.GUID; import org.alfresco.util.ISO9075; import org.alfresco.util.Pair; import org.apache.log4j.Logger; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Searcher; /** * Update the index after a snap shot to an AVM store. (Revert is dealt with as a new snap shot is created) @@ -85,6 +94,10 @@ import org.apache.lucene.document.Field; */ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl implements AVMLuceneIndexer { + private static String SNAP_SHOT_ID = "SnapShotId_"; + + private static String SNAP_SHOT_STORE = "SnapShotStore"; + static Logger s_logger = Logger.getLogger(AVMLuceneIndexerImpl.class); private AVMService avmService; @@ -98,8 +111,19 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl private Set indexedPaths = new HashSet(); + private FTSIndexerAware callBack; + + private FullTextSearchIndexer fullTextSearchIndexer; + + private int remainingCount; + + private int startVersion = -1; + + private int endVersion = -1; + /** * Set the AVM Service + * * @param avmService */ public void setAvmService(AVMService avmService) @@ -109,6 +133,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl /** * Set the AVM sync service + * * @param avmSyncService */ public void setAvmSyncService(AVMSyncService avmSyncService) @@ -118,6 +143,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl /** * Set the content service + * * @param contentStore */ public void setContentStore(ContentStore contentStore) @@ -162,10 +188,55 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl * @param srcVersion * @param dstVersion */ - public void index(String store, int srcVersion, int dstVersion) + public void index(String store, int srcVersion, int dstVersion, IndexMode mode) { checkAbleToDoWork(IndexUpdateStatus.SYNCRONOUS); + try + { + switch (mode) + { + case ASYNCHRONOUS: + asynchronousIndex(store, srcVersion, dstVersion); + break; + case SYNCHRONOUS: + synchronousIndex(store, srcVersion, dstVersion); + break; + case UNINDEXED: + // nothing to do + break; + } + } + catch (LuceneIndexException e) + { + setRollbackOnly(); + throw new LuceneIndexException("snapshot index failed", e); + } + } + public void asynchronousIndex(String store, int srcVersion, int dstVersion) + { + if(s_logger.isDebugEnabled()) + { + s_logger.debug("Async index for "+store + " from " + srcVersion + " to " + dstVersion); + } + index("\u0000BG:STORE:" + store + ":" + srcVersion + ":" + dstVersion + ":" + GUID.generate()); + fullTextSearchIndexer.requiresIndex(AVMNodeConverter.ToStoreRef(store)); + } + + public void synchronousIndex(String store, int srcVersion, int dstVersion) + { + if(startVersion == -1) + { + startVersion = srcVersion; + } + + endVersion = dstVersion; + + + if(s_logger.isDebugEnabled()) + { + s_logger.debug("Sync index for "+store + " from " + srcVersion + " to "+dstVersion); + } String path = store + ":/"; List changeList = avmSyncService.compare(srcVersion, path, dstVersion, path, null); for (AVMDifference difference : changeList) @@ -221,6 +292,8 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl } } + // record the snap shotid + reindex(SNAP_SHOT_ID + store, false); } /* @@ -232,14 +305,14 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl String store = splitPath[0]; String pathInStore = splitPath[1]; SimplePath simplePath = new SimplePath(pathInStore); - + StringBuilder pathBuilder = new StringBuilder(); pathBuilder.append(store).append(":/"); reindex(pathBuilder.toString(), false); boolean requiresSep = false; for (int i = 0; i < simplePath.size() - 1; i++) { - if(requiresSep) + if (requiresSep) { pathBuilder.append("/"); } @@ -270,25 +343,64 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl protected List createDocuments(String stringNodeRef, boolean isNew, boolean indexAllProperties, boolean includeDirectoryDocuments) { - AVMNodeDescriptor desc = avmService.lookup(-1, stringNodeRef); List docs = new ArrayList(); - if(desc == null) + if (stringNodeRef.startsWith("\u0000")) + { + Document idoc = new Document(); + idoc.add(new Field("ID", stringNodeRef, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); + docs.add(idoc); + return docs; + } + else if (stringNodeRef.startsWith(SNAP_SHOT_ID)) + { + Document sdoc = new Document(); + sdoc.add(new Field("ID", stringNodeRef, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); + String storeName = stringNodeRef.substring(SNAP_SHOT_ID.length()); + sdoc.add(new Field(SNAP_SHOT_ID, NumericEncoder.encode(avmService.getLatestSnapshotID(storeName)), + Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); + sdoc.add(new Field(SNAP_SHOT_STORE, NumericEncoder.encode(avmService.getLatestSnapshotID(storeName)), + Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); + docs.add(sdoc); + return docs; + } + + AVMNodeDescriptor desc = avmService.lookup(endVersion, stringNodeRef); + if (desc == null) { return docs; } if (desc.isLayeredDirectory() || desc.isLayeredFile()) { return docs; - } - - List> paths = avmService.getHeadPaths(desc); + } + + List> allPaths = avmService.getPaths(desc); + List> paths = new ArrayList> (); + for(Pair pair: allPaths) + { + if(pair.getFirst().intValue() == endVersion) + { + paths.add(pair); + } + } if(paths.size() == 0) + { + for(Pair pair: allPaths) + { + if(pair.getFirst().intValue() == -1) + { + paths.add(pair); + } + } + } + + if (paths.size() == 0) { return docs; } for (Pair path : paths) { - if(indexedPaths.contains(path.getSecond())) + if (indexedPaths.contains(path.getSecond())) { return docs; } @@ -297,37 +409,35 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl { indexedPaths.add(path.getSecond()); } - + AVMNode node = AVMDAOs.Instance().fAVMNodeDAO.getByID(desc.getId()); - if (paths.size() > 0) { if (desc != null) { - NodeRef nodeRef = AVMNodeConverter.ToNodeRef(-1, stringNodeRef); + NodeRef nodeRef = AVMNodeConverter.ToNodeRef(endVersion, stringNodeRef); Document xdoc = new Document(); - xdoc.add(new Field("ID", nodeRef.toString(), Field.Store.YES, - Field.Index.UN_TOKENIZED, Field.TermVector.NO)); + xdoc.add(new Field("ID", nodeRef.toString(), Field.Store.YES, Field.Index.UN_TOKENIZED, + Field.TermVector.NO)); for (Pair path : paths) { String[] splitPath = splitPath(path.getSecond()); @SuppressWarnings("unused") String pathInStore = splitPath[1]; - xdoc.add(new Field("ID", path.getSecond(), - Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); + xdoc.add(new Field("ID", path.getSecond(), Field.Store.YES, Field.Index.UN_TOKENIZED, + Field.TermVector.NO)); } - xdoc.add(new Field("TX", AlfrescoTransactionSupport.getTransactionId(), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); boolean isAtomic = true; - Map properties = getIndexableProperties(desc, node, nodeRef, -1, stringNodeRef); + Map properties = getIndexableProperties(desc, node, nodeRef, endVersion, stringNodeRef); for (QName propertyName : properties.keySet()) { Serializable value = properties.get(propertyName); @@ -357,9 +467,9 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl String store = splitPath[0]; String pathInStore = splitPath[1]; SimplePath simplePath = new SimplePath(pathInStore); - + StringBuilder xpathBuilder = new StringBuilder(); - for(int i = 0; i < simplePath.size(); i++) + for (int i = 0; i < simplePath.size(); i++) { xpathBuilder.append("/{}").append(simplePath.get(i)); } @@ -379,7 +489,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl boolean requiresSep = false; for (int i = 0; i < simplePath.size() - 1; i++) { - if(requiresSep) + if (requiresSep) { pathBuilder.append("/"); } @@ -393,8 +503,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl qNameBuffer.append(ISO9075.getXPathName(QName.createQName("", simplePath .get(simplePath.size() - 1)))); - xdoc.add(new Field("PARENT", - ancestors.get(ancestors.size() - 1), Field.Store.YES, + xdoc.add(new Field("PARENT", ancestors.get(ancestors.size() - 1), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); // TODO: Categories and LINKASPECT @@ -407,20 +516,19 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl Document directoryEntry = new Document(); directoryEntry.add(new Field("ID", nodeRef.toString(), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); - - directoryEntry.add(new Field("ID", path.getSecond(), - Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); - - - directoryEntry.add(new Field("PATH", xpath, Field.Store.YES, - Field.Index.TOKENIZED, Field.TermVector.NO)); + + directoryEntry.add(new Field("ID", path.getSecond(), Field.Store.YES, + Field.Index.UN_TOKENIZED, Field.TermVector.NO)); + + directoryEntry.add(new Field("PATH", xpath, Field.Store.YES, Field.Index.TOKENIZED, + Field.TermVector.NO)); // Find all parent nodes. for (String toAdd : ancestors) { - directoryEntry.add(new Field("ANCESTOR", toAdd, - Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); + directoryEntry.add(new Field("ANCESTOR", toAdd, Field.Store.NO, + Field.Index.UN_TOKENIZED, Field.TermVector.NO)); } directoryEntry.add(new Field("ISCONTAINER", "T", Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); @@ -456,7 +564,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl xdoc.add(new Field("TYPE", ISO9075.getXPathName(typeQName), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); - for (QName classRef : avmService.getAspects(-1, stringNodeRef)) + for (QName classRef : avmService.getAspects(endVersion, stringNodeRef)) { xdoc.add(new Field("ASPECT", ISO9075.getXPathName(classRef), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO)); @@ -534,11 +642,11 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl result.put(ContentModel.PROP_STORE_IDENTIFIER, nodeRef.getStoreRef().getIdentifier()); if (desc.isLayeredDirectory()) { - result.put(WCMModel.PROP_AVM_DIR_INDIRECTION, AVMNodeConverter.ToNodeRef(-1, desc.getIndirection())); + result.put(WCMModel.PROP_AVM_DIR_INDIRECTION, AVMNodeConverter.ToNodeRef(endVersion, desc.getIndirection())); } if (desc.isLayeredFile()) { - result.put(WCMModel.PROP_AVM_FILE_INDIRECTION, AVMNodeConverter.ToNodeRef(-1, desc.getIndirection())); + result.put(WCMModel.PROP_AVM_FILE_INDIRECTION, AVMNodeConverter.ToNodeRef(endVersion, desc.getIndirection())); } if (desc.isFile()) { @@ -873,13 +981,32 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl @Override protected void doCommit() throws IOException { + if (indexUpdateStatus == IndexUpdateStatus.ASYNCHRONOUS) + { + setInfo(docs, getDeletions(), false); + // FTS does not trigger indexing request + } + else + { + setInfo(docs, getDeletions(), false); + // TODO: only register if required + fullTextSearchIndexer.requiresIndex(store); + } + if (callBack != null) + { + callBack.indexCompleted(store, remainingCount, null); + } + setInfo(docs, deletions, false); } @Override protected void doRollBack() throws IOException { - + if (callBack != null) + { + callBack.indexCompleted(store, 0, null); + } } @Override @@ -1014,29 +1141,68 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl } - public void deleteIndex(String store) + public void deleteIndex(String store, IndexMode mode) { checkAbleToDoWork(IndexUpdateStatus.SYNCRONOUS); try { - deleteAll(); + switch (mode) + { + case ASYNCHRONOUS: + asyncronousDeleteIndex(store); + break; + case SYNCHRONOUS: + syncronousDeleteIndex(store); + break; + case UNINDEXED: + // nothing to do + break; + } } catch (LuceneIndexException e) { setRollbackOnly(); throw new LuceneIndexException("Delete index failed", e); } - } - public void createIndex(String store) + public void syncronousDeleteIndex(String store) + { + + if(s_logger.isDebugEnabled()) + { + s_logger.debug("Sync delete for "+store); + } + deleteAll(); + } + + public void asyncronousDeleteIndex(String store) + { + if(s_logger.isDebugEnabled()) + { + s_logger.debug("Async delete for "+store); + } + index("\u0000BG:DELETE:" + store+":"+GUID.generate()); + fullTextSearchIndexer.requiresIndex(AVMNodeConverter.ToStoreRef(store)); + } + + public void createIndex(String store, IndexMode mode) { checkAbleToDoWork(IndexUpdateStatus.SYNCRONOUS); try { - @SuppressWarnings("unused") - AVMNodeDescriptor rootDesc = avmService.getStoreRoot(-1, store); - index(store+":/"); + switch (mode) + { + case ASYNCHRONOUS: + asyncronousCreateIndex(store); + break; + case SYNCHRONOUS: + syncronousCreateIndex(store); + break; + case UNINDEXED: + // nothing to do + break; + } } catch (LuceneIndexException e) { @@ -1044,4 +1210,123 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl impl throw new LuceneIndexException("Create index failed", e); } } + + public void syncronousCreateIndex(String store) + { + + if(s_logger.isDebugEnabled()) + { + s_logger.debug("Sync create for "+store); + } + @SuppressWarnings("unused") + AVMNodeDescriptor rootDesc = avmService.getStoreRoot(-1, store); + index(store + ":/"); + + } + + public void asyncronousCreateIndex(String store) + { + if(s_logger.isDebugEnabled()) + { + s_logger.debug("Async create for "+store); + } + index("\u0000BG:CREATE:" + store+":"+GUID.generate()); + fullTextSearchIndexer.requiresIndex(AVMNodeConverter.ToStoreRef(store)); + } + + public void registerCallBack(FTSIndexerAware callBack) + { + this.callBack = callBack; + } + + public int updateFullTextSearch(int size) + { + checkAbleToDoWork(IndexUpdateStatus.ASYNCHRONOUS); + + try + { + PrefixQuery query = new PrefixQuery(new Term("ID", "\u0000BG:")); + + String action = null; + + Searcher searcher = null; + try + { + searcher = getSearcher(null); + // commit on another thread - appears like there is no index ...try later + if (searcher == null) + { + remainingCount = size; + return 0; + } + Hits hits; + try + { + hits = searcher.search(query); + } + catch (IOException e) + { + throw new LuceneIndexException( + "Failed to execute query to find content which needs updating in the index", e); + } + + if (hits.length() > 0) + { + Document doc = hits.doc(0); + action = doc.getField("ID").stringValue(); + String[] split = action.split(":"); + if(split[1].equals("DELETE")) + { + deleteAll("\u0000BG:"); + } + else if(split[1].equals("CREATE")) + { + syncronousCreateIndex(split[2]); + } + else if(split[1].equals("STORE")) + { + synchronousIndex(split[2], Integer.parseInt(split[3]), Integer.parseInt(split[4])); + } + deletions.add(action); + remainingCount = hits.length() - 1; + return 1; + } + else + { + remainingCount = 0; + return 0; + } + + } + finally + { + if (searcher != null) + { + try + { + searcher.close(); + } + catch (IOException e) + { + throw new LuceneIndexException("Failed to close searcher", e); + } + } + } + } + catch (IOException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Failed FTS update", e); + } + catch (LuceneIndexException e) + { + setRollbackOnly(); + throw new LuceneIndexException("Failed FTS update", e); + } + } + + public void setFullTextSearchIndexer(FullTextSearchIndexer fullTextSearchIndexer) + { + this.fullTextSearchIndexer = fullTextSearchIndexer; + } } diff --git a/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerImpl.java b/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerImpl.java index c2257b4811..dfa1b2e3da 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerImpl.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerImpl.java @@ -63,15 +63,15 @@ public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase /** * An index */ - INDEX, + INDEX, /** * A reindex */ - REINDEX, + REINDEX, /** * A delete */ - DELETE, + DELETE, /** * A cascaded reindex (ensures directory structre is ok) */ @@ -83,13 +83,13 @@ public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase /** * Inde is unchanged */ - UNMODIFIED, + UNMODIFIED, /** - * Index is being changein in TX + * Index is being changein in TX */ - SYNCRONOUS, + SYNCRONOUS, /** - * Index is eiong changed by a background upate + * Index is eiong changed by a background upate */ ASYNCHRONOUS; } @@ -379,7 +379,8 @@ public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase /** * Commit this index - * @throws LuceneIndexException + * + * @throws LuceneIndexException */ public void commit() throws LuceneIndexException { @@ -441,7 +442,7 @@ public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase * serialisation against the index as would a data base transaction. * * @return the tx state - * @throws LuceneIndexException + * @throws LuceneIndexException */ public int prepare() throws LuceneIndexException { @@ -498,7 +499,8 @@ public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase /** * Roll back the index changes (this just means they are never added) - * @throws LuceneIndexException + * + * @throws LuceneIndexException */ public void rollback() throws LuceneIndexException { @@ -819,6 +821,7 @@ public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase /** * Are we deleting leaves only (not meta data) + * * @return - deleting only nodes. */ public boolean getDeleteOnlyNodes() @@ -828,6 +831,7 @@ public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase /** * Get the deletions + * * @return - the ids to delete */ public Set getDeletions() @@ -837,9 +841,18 @@ public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase /** * Delete all entries from the index. - * */ public void deleteAll() + { + deleteAll(null); + } + + /** + * Delete all index entries which do not start with the goven prefix + * + * @param prefix + */ + public void deleteAll(String prefix) { IndexReader mainReader = null; try @@ -851,7 +864,10 @@ public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase { Document document = mainReader.document(doc); String[] ids = document.getValues("ID"); - deletions.add(ids[ids.length - 1]); + if ((prefix == null) || nonStartwWith(ids, prefix)) + { + deletions.add(ids[ids.length - 1]); + } } } @@ -877,4 +893,16 @@ public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase } } + private boolean nonStartwWith(String[] values, String prefix) + { + for (String value : values) + { + if (value.startsWith(prefix)) + { + return false; + } + } + return true; + } + }