inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-07-24 17:32:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Asynchronous indexing for AVM

Browse Source 
git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@5795 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Andrew Hind
2007-05-29 10:19:57 +00:00
parent c5e4a60cd7
commit 43dad1aeff
8 changed files with 723 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
source/java/org/alfresco/repo/avm/AVMServiceTest.java
View File

@@ -37,6 +37,10 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import javax.transaction.NotSupportedException;
import javax.transaction.SystemException;
import javax.transaction.UserTransaction;
import org.alfresco.config.JNDIConstants;
import org.alfresco.model.ContentModel;
import org.alfresco.model.WCMModel;
@@ -49,6 +53,9 @@ import org.alfresco.repo.avm.actions.SimpleAVMSubmitAction;
import org.alfresco.repo.avm.util.BulkLoader;
import org.alfresco.repo.content.MimetypeMap;
import org.alfresco.repo.domain.PropertyValue;
import org.alfresco.repo.search.IndexMode;
import org.alfresco.repo.search.Indexer;
import org.alfresco.repo.search.impl.lucene.AVMLuceneIndexer;
import org.alfresco.repo.search.impl.lucene.LuceneQueryParser;
import org.alfresco.repo.search.impl.lucene.analysis.NumericEncoder;
import org.alfresco.repo.transaction.RetryingTransactionHelper;
@@ -94,6 +101,86 @@ import org.alfresco.util.Pair;
*/
public class AVMServiceTest extends AVMServiceTestBase
{
/**
* Test async indexing.
* @throws Exception
*/
public void testAsyncIndex() throws Exception
{
// Make sure the slate is clean ...
UserTransaction tx = fTransactionService.getUserTransaction();
tx.begin();
if(fService.getStore("avmAsynchronousTest") != null)
{
fService.purgeStore("avmAsynchronousTest");
}
StoreRef storeRef = AVMNodeConverter.ToStoreRef("avmAsynchronousTest");
Indexer indexer = fIndexerAndSearcher.getIndexer(storeRef);
if(indexer instanceof AVMLuceneIndexer)
{
AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer)indexer;
avmIndexer.deleteIndex("avmAsynchronousTest", IndexMode.SYNCHRONOUS);
}
tx.commit();
// TODO: Suspend and resume indexing in case we are really unlucky and hit an index before we expect it.
SearchService searchService = fIndexerAndSearcher.getSearcher(storeRef, true);
ResultSet results;
results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
assertEquals(0, results.length());
results.close();
fService.createStore("avmAsynchronousTest");
fService.createSnapshot("avmAsynchronousTest", null, null);
results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
assertEquals(1, results.length());
results.close();
fService.createDirectory("avmAsynchronousTest:/", "a");
fService.createDirectory("avmAsynchronousTest:/a", "b");
fService.createDirectory("avmAsynchronousTest:/a/b", "c");
fService.createSnapshot("avmAsynchronousTest", null, null);
results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
assertEquals(1, results.length());
results.close();
Thread.sleep(120000);
results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
assertEquals(4, results.length());
results.close();
fService.purgeStore("avmAsynchronousTest");
results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
assertEquals(0, results.length());
results.close();
fService.createStore("avmAsynchronousTest");
fService.createSnapshot("avmAsynchronousTest", null, null);
fService.createDirectory("avmAsynchronousTest:/", "a");
fService.createDirectory("avmAsynchronousTest:/a", "b");
fService.createDirectory("avmAsynchronousTest:/a/b", "c");
fService.createSnapshot("avmAsynchronousTest", null, null);
fService.purgeStore("avmAsynchronousTest");
fService.createStore("avmAsynchronousTest");
fService.createSnapshot("avmAsynchronousTest", null, null);
fService.createDirectory("avmAsynchronousTest:/", "a");
fService.createDirectory("avmAsynchronousTest:/a", "b");
fService.createDirectory("avmAsynchronousTest:/a/b", "c");
fService.createSnapshot("avmAsynchronousTest", null, null);
Thread.sleep(120000);
results = searchService.query(storeRef, "lucene", "PATH:\"//.\"");
assertEquals(4, results.length());
results.close();
}
public void testForceCopyDeleted()
{
try
@@ -125,7 +212,6 @@ public class AVMServiceTest extends AVMServiceTestBase
runQueriesAgainstBasicTree("main");
fService.createFile("main:/a", "Xander");
fService.createSnapshot("layer", null, null);
runQueriesAgainstBasicTree("main");
assertEquals(2, fService.lookup(2, "layer:/a").getIndirectionVersion());
assertEquals(fService.lookup(2, "main:/a/Xander").getId(), fService.lookup(2, "layer:/a/Xander").getId());
assertNull(fService.lookup(1, "layer:/a/Xander"));

4
source/java/org/alfresco/repo/avm/AVMServiceTestBase.java
View File

@@ -45,6 +45,7 @@ import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.service.cmr.search.SearchService;
import org.alfresco.service.cmr.security.AuthenticationService;
import org.alfresco.service.transaction.TransactionService;
import org.springframework.context.support.FileSystemXmlApplicationContext;
import junit.framework.TestCase;
@@ -79,6 +80,8 @@ public class AVMServiceTestBase extends TestCase
*/
private long fStartTime;
protected TransactionService fTransactionService;
protected static IndexerAndSearcher fIndexerAndSearcher;
/**
@@ -96,6 +99,7 @@ public class AVMServiceTestBase extends TestCase
fReaper = (OrphanReaper)fContext.getBean("orphanReaper");
fSyncService = (AVMSyncService)fContext.getBean("AVMSyncService");
fIndexerAndSearcher = (IndexerAndSearcher)fContext.getBean("indexerAndSearcherFactory");
fTransactionService = (TransactionService)fContext.getBean("transactionComponent");
AuthenticationService authService = (AuthenticationService)fContext.getBean("AuthenticationService");
authService.authenticate("admin", "admin".toCharArray());
CreateStoreTxnListener cstl = (CreateStoreTxnListener)fContext.getBean("createStoreTxnListener");

237
source/java/org/alfresco/repo/search/AVMSnapShotTriggeredIndexingMethodInterceptor.java
View File

@@ -24,46 +24,109 @@
*/
package org.alfresco.repo.search;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.repo.avm.AVMNodeConverter;
import org.alfresco.repo.domain.PropertyValue;
import org.alfresco.repo.search.impl.lucene.AVMLuceneIndexer;
import org.alfresco.service.cmr.avm.AVMService;
import org.alfresco.service.cmr.avm.AVMStoreDescriptor;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.namespace.QName;
import org.aopalliance.intercept.MethodInterceptor;
import org.aopalliance.intercept.MethodInvocation;
/**
* Method interceptor for atomic indexing of AVM entries
*
* The proeprties can defined how stores are indexed based on type (as set by Alfresco the Web site management UI)
* or based on the name of the store.
*
* Creates and deletes are indexed synchronously.
*
* Updates may be asynchronous, synchronous or ignored by the index.
*
* @author andyh
*/
public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInterceptor
{
// Copy of store properties used to tag avm stores (a store propertry)
public final static QName PROP_SANDBOX_STAGING_MAIN = QName.createQName(null, ".sandbox.staging.main");
public final static QName PROP_SANDBOX_STAGING_PREVIEW = QName.createQName(null, ".sandbox.staging.preview");
public final static QName PROP_SANDBOX_AUTHOR_MAIN = QName.createQName(null, ".sandbox.author.main");
public final static QName PROP_SANDBOX_AUTHOR_PREVIEW = QName.createQName(null, ".sandbox.author.preview");
public final static QName PROP_SANDBOX_WORKFLOW_MAIN = QName.createQName(null, ".sandbox.workflow.main");
public final static QName PROP_SANDBOX_WORKFLOW_PREVIEW = QName.createQName(null, ".sandbox.workflow.preview");
public final static QName PROP_SANDBOX_AUTHOR_WORKFLOW_MAIN = QName.createQName(null,
".sandbox.author.workflow.main");
public final static QName PROP_SANDBOX_AUTHOR_WORKFLOW_PREVIEW = QName.createQName(null,
".sandbox.author.workflow.preview");
private AVMService avmService;
private IndexerAndSearcher indexerAndSearcher;
private boolean enableIndexing = true;
private IndexMode defaultMode = IndexMode.ASYNCHRONOUS;
private Map<String, IndexMode> modeCache = new HashMap<String, IndexMode>();
private List<IndexingDefinition> indexingDefinitions = new ArrayList<IndexingDefinition>();
public Object invoke(MethodInvocation mi) throws Throwable
{
if (enableIndexing)
{
if (mi.getMethod().getName().equals("createSnapshot"))
{
String store = (String) mi.getArguments()[0];
int before = avmService.getLatestSnapshotID(store);
Object returnValue = mi.proceed();
int after = avmService.getLatestSnapshotID(store);
StoreRef storeRef = AVMNodeConverter.ToStoreRef(store);
Indexer indexer = indexerAndSearcher.getIndexer(storeRef);
if (indexer instanceof AVMLuceneIndexer)
// May cause any number of other stores to do snap shot under the covers via layering or do nothing
// So we have to watch what actually changes
List<AVMStoreDescriptor> stores = avmService.getStores();
Map<String, Integer> initialStates = new HashMap<String, Integer>(stores.size(), 1.0f);
for (AVMStoreDescriptor desc : stores)
{
AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
avmIndexer.index(store, before, after);
String store = desc.getName();
Integer state = Integer.valueOf(avmService.getLatestSnapshotID(store));
initialStates.put(store, state);
}
Object returnValue = mi.proceed();
// Index any stores that have moved on
for (AVMStoreDescriptor desc : stores)
{
String store = desc.getName();
int after = avmService.getLatestSnapshotID(store);
int before = initialStates.get(store).intValue();
if (after > before)
{
StoreRef storeRef = AVMNodeConverter.ToStoreRef(store);
Indexer indexer = indexerAndSearcher.getIndexer(storeRef);
if (indexer instanceof AVMLuceneIndexer)
{
AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
avmIndexer.index(store, before, after, getIndexMode(store));
}
}
}
return returnValue;
}
// TODO: Purge store
else if (mi.getMethod().getName().equals("purgeStore"))
{
String store = (String) mi.getArguments()[0];
@@ -73,7 +136,7 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte
if (indexer instanceof AVMLuceneIndexer)
{
AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
avmIndexer.deleteIndex(store);
avmIndexer.deleteIndex(store, IndexMode.SYNCHRONOUS);
}
return returnValue;
}
@@ -86,7 +149,7 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte
if (indexer instanceof AVMLuceneIndexer)
{
AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
avmIndexer.createIndex(store);
avmIndexer.createIndex(store, IndexMode.SYNCHRONOUS);
}
return returnValue;
}
@@ -104,15 +167,15 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte
if (indexer instanceof AVMLuceneIndexer)
{
AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
avmIndexer.deleteIndex(from);
avmIndexer.deleteIndex(from, IndexMode.SYNCHRONOUS);
}
indexer = indexerAndSearcher.getIndexer(toRef);
if (indexer instanceof AVMLuceneIndexer)
{
AVMLuceneIndexer avmIndexer = (AVMLuceneIndexer) indexer;
avmIndexer.createIndex(to);
avmIndexer.index(to, 0, after);
avmIndexer.createIndex(to, IndexMode.SYNCHRONOUS);
avmIndexer.index(to, 0, after, getIndexMode(to));
}
return returnValue;
@@ -157,7 +220,147 @@ public class AVMSnapShotTriggeredIndexingMethodInterceptor implements MethodInte
{
this.enableIndexing = enableIndexing;
}
/**
* Set the index modes.... Strings of the form ... (ASYNCHRONOUS | SYNCHRONOUS | UNINDEXED):(NAME | TYPE):regexp
*
* @param definitions
*/
public void setIndexingDefinitions(List<String> definitions)
{
indexingDefinitions.clear();
for (String def : definitions)
{
IndexingDefinition id = new IndexingDefinition(def);
indexingDefinitions.add(id);
}
}
/**
* Set the default index mode = used when there are no matches
*
* @param defaultMode
*/
public void setDefaultMode(IndexMode defaultMode)
{
this.defaultMode = defaultMode;
}
private synchronized IndexMode getIndexMode(String store)
{
IndexMode mode = modeCache.get(store);
if (mode == null)
{
for (IndexingDefinition def : indexingDefinitions)
{
if (def.definitionType == DefinitionType.NAME)
{
if (def.pattern.matcher(store).matches())
{
mode = def.indexMode;
modeCache.put(store, mode);
break;
}
}
else
{
String storeType = getStoreType(store).toString();
if (def.pattern.matcher(storeType).matches())
{
mode = def.indexMode;
modeCache.put(store, mode);
break;
}
}
}
}
// No definition
if (mode == null)
{
mode = defaultMode;
modeCache.put(store, mode);
}
return mode;
}
private class IndexingDefinition
{
IndexMode indexMode;
DefinitionType definitionType;
Pattern pattern;
IndexingDefinition(String definition)
{
String[] split = definition.split(":", 3);
if (split.length != 3)
{
throw new AlfrescoRuntimeException(
"Invalid index defintion. Must be of of the form IndexMode:DefinitionType:regular expression");
}
indexMode = IndexMode.valueOf(split[0].toUpperCase());
definitionType = DefinitionType.valueOf(split[1].toUpperCase());
pattern = Pattern.compile(split[2]);
}
}
private StoreType getStoreType(String name)
{
if (avmService.getStore(name) != null)
{
Map<QName, PropertyValue> storeProperties = avmService.getStoreProperties(name);
if (storeProperties.containsKey(PROP_SANDBOX_STAGING_MAIN))
{
return StoreType.STAGING;
}
else if (storeProperties.containsKey(PROP_SANDBOX_STAGING_PREVIEW))
{
return StoreType.STAGING_PREVIEW;
}
else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_MAIN))
{
return StoreType.AUTHOR;
}
else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_PREVIEW))
{
return StoreType.AUTHOR_PREVIEW;
}
else if (storeProperties.containsKey(PROP_SANDBOX_WORKFLOW_MAIN))
{
return StoreType.WORKFLOW;
}
else if (storeProperties.containsKey(PROP_SANDBOX_WORKFLOW_PREVIEW))
{
return StoreType.WORKFLOW_PREVIEW;
}
else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_WORKFLOW_MAIN))
{
return StoreType.AUTHOR_WORKFLOW;
}
else if (storeProperties.containsKey(PROP_SANDBOX_AUTHOR_WORKFLOW_PREVIEW))
{
return StoreType.AUTHOR_WORKFLOW_PREVIEW;
}
else
{
return StoreType.UNKNOWN;
}
}
else
{
return StoreType.UNKNOWN;
}
}
private enum DefinitionType
{
NAME, TYPE;
}
private enum StoreType
{
STAGING, STAGING_PREVIEW, AUTHOR, AUTHOR_PREVIEW, WORKFLOW, WORKFLOW_PREVIEW, AUTHOR_WORKFLOW, AUTHOR_WORKFLOW_PREVIEW, UNKNOWN;
}
}

14
source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexer.java
View File

@@ -24,13 +24,16 @@
*/
package org.alfresco.repo.search.impl.lucene;
import org.alfresco.repo.search.BackgroundIndexerAware;
import org.alfresco.repo.search.IndexMode;
/**
* AVM specific indxer support
*
* @author andyh
*
*/
public interface AVMLuceneIndexer extends LuceneIndexer
public interface AVMLuceneIndexer extends LuceneIndexer, BackgroundIndexerAware
{
/**
* Index a specified change to a store between two snapshots
@@ -38,21 +41,24 @@ public interface AVMLuceneIndexer extends LuceneIndexer
* @param store - the name of the store
* @param srcVersion - the id of the snapshot before the changeset
* @param dstVersion - the id of the snapshot created by the change set
* @param mode
*/
public void index(String store, int srcVersion, int dstVersion);
public void index(String store, int srcVersion, int dstVersion, IndexMode mode);
/**
* Delete the index for the specified store.
*
* @param store
* @param mode
*/
public void deleteIndex(String store);
public void deleteIndex(String store, IndexMode mode);
/**
* Create an index for the specified store.
* This makes sure that the root node for the store is indexed correctly.
*
* @param store
* @param mode
*/
public void createIndex(String store);
public void createIndex(String store, IndexMode mode);
}

16
source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerAndSearcherFactory.java
View File

@@ -30,6 +30,8 @@ import java.util.List;
import org.alfresco.repo.avm.AVMNodeConverter;
import org.alfresco.repo.content.ContentStore;
import org.alfresco.repo.search.SearcherException;
import org.alfresco.repo.search.SupportsBackgroundIndexing;
import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer;
import org.alfresco.service.cmr.avm.AVMService;
import org.alfresco.service.cmr.avm.AVMStoreDescriptor;
import org.alfresco.service.cmr.avmsync.AVMSyncService;
@@ -47,7 +49,7 @@ import org.apache.commons.logging.LogFactory;
* @author andyh
*
*/
public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAndSearcherFactory
public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAndSearcherFactory implements SupportsBackgroundIndexing
{
private static Log s_logger = LogFactory.getLog(AVMLuceneIndexerAndSearcherFactory.class);
@@ -65,6 +67,8 @@ public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAnd
private ContentStore contentStore;
private FullTextSearchIndexer fullTextSearchIndexer;
public AVMLuceneIndexerAndSearcherFactory()
{
//s_logger.error("Creating AVMLuceneIndexerAndSearcherFactory");
@@ -145,6 +149,7 @@ public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAnd
indexer.setAvmService(avmService);
indexer.setAvmSyncService(avmSyncService);
indexer.setContentStore(contentStore);
indexer.setFullTextSearchIndexer(fullTextSearchIndexer);
return indexer;
}
@@ -174,4 +179,13 @@ public class AVMLuceneIndexerAndSearcherFactory extends AbstractLuceneIndexerAnd
return searcher;
}
/**
* Register the full text searcher (done by the seracher bean to break cyclic bean defs)
*
*/
public void setFullTextSearchIndexer(FullTextSearchIndexer fullTextSearchIndexer)
{
this.fullTextSearchIndexer = fullTextSearchIndexer;
}
}

371
source/java/org/alfresco/repo/search/impl/lucene/AVMLuceneIndexerImpl.java
View File

@@ -51,6 +51,10 @@ import org.alfresco.repo.content.ContentStore;
import org.alfresco.repo.content.MimetypeMap;
import org.alfresco.repo.content.transform.ContentTransformer;
import org.alfresco.repo.domain.PropertyValue;
import org.alfresco.repo.search.IndexMode;
import org.alfresco.repo.search.impl.lucene.analysis.NumericEncoder;
import org.alfresco.repo.search.impl.lucene.fts.FTSIndexerAware;
import org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer;
import org.alfresco.repo.transaction.AlfrescoTransactionSupport;
import org.alfresco.service.cmr.avm.AVMException;
import org.alfresco.service.cmr.avm.AVMNodeDescriptor;
@@ -72,11 +76,16 @@ import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
import org.alfresco.service.cmr.repository.datatype.TypeConversionException;
import org.alfresco.service.namespace.QName;
import org.alfresco.util.EqualsHelper;
import org.alfresco.util.GUID;
import org.alfresco.util.ISO9075;
import org.alfresco.util.Pair;
import org.apache.log4j.Logger;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Searcher;
/**
* Update the index after a snap shot to an AVM store. (Revert is dealt with as a new snap shot is created)
@@ -85,6 +94,10 @@ import org.apache.lucene.document.Field;
*/
public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> implements AVMLuceneIndexer
{
private static String SNAP_SHOT_ID = "SnapShotId_";
private static String SNAP_SHOT_STORE = "SnapShotStore";
static Logger s_logger = Logger.getLogger(AVMLuceneIndexerImpl.class);
private AVMService avmService;
@@ -98,8 +111,19 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
private Set<String> indexedPaths = new HashSet<String>();
private FTSIndexerAware callBack;
private FullTextSearchIndexer fullTextSearchIndexer;
private int remainingCount;
private int startVersion = -1;
private int endVersion = -1;
/**
* Set the AVM Service
*
* @param avmService
*/
public void setAvmService(AVMService avmService)
@@ -109,6 +133,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
/**
* Set the AVM sync service
*
* @param avmSyncService
*/
public void setAvmSyncService(AVMSyncService avmSyncService)
@@ -118,6 +143,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
/**
* Set the content service
*
* @param contentStore
*/
public void setContentStore(ContentStore contentStore)
@@ -162,10 +188,55 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
* @param srcVersion
* @param dstVersion
*/
public void index(String store, int srcVersion, int dstVersion)
public void index(String store, int srcVersion, int dstVersion, IndexMode mode)
{
checkAbleToDoWork(IndexUpdateStatus.SYNCRONOUS);
try
{
switch (mode)
{
case ASYNCHRONOUS:
asynchronousIndex(store, srcVersion, dstVersion);
break;
case SYNCHRONOUS:
synchronousIndex(store, srcVersion, dstVersion);
break;
case UNINDEXED:
// nothing to do
break;
}
}
catch (LuceneIndexException e)
{
setRollbackOnly();
throw new LuceneIndexException("snapshot index failed", e);
}
}
public void asynchronousIndex(String store, int srcVersion, int dstVersion)
{
if(s_logger.isDebugEnabled())
{
s_logger.debug("Async index for "+store + " from " + srcVersion + " to " + dstVersion);
}
index("\u0000BG:STORE:" + store + ":" + srcVersion + ":" + dstVersion + ":" + GUID.generate());
fullTextSearchIndexer.requiresIndex(AVMNodeConverter.ToStoreRef(store));
}
public void synchronousIndex(String store, int srcVersion, int dstVersion)
{
if(startVersion == -1)
{
startVersion = srcVersion;
}
endVersion = dstVersion;
if(s_logger.isDebugEnabled())
{
s_logger.debug("Sync index for "+store + " from " + srcVersion + " to "+dstVersion);
}
String path = store + ":/";
List<AVMDifference> changeList = avmSyncService.compare(srcVersion, path, dstVersion, path, null);
for (AVMDifference difference : changeList)
@@ -221,6 +292,8 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
}
}
// record the snap shotid
reindex(SNAP_SHOT_ID + store, false);
}
/*
@@ -232,14 +305,14 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
String store = splitPath[0];
String pathInStore = splitPath[1];
SimplePath simplePath = new SimplePath(pathInStore);
StringBuilder pathBuilder = new StringBuilder();
pathBuilder.append(store).append(":/");
reindex(pathBuilder.toString(), false);
boolean requiresSep = false;
for (int i = 0; i < simplePath.size() - 1; i++)
{
if(requiresSep)
if (requiresSep)
{
pathBuilder.append("/");
}
@@ -270,25 +343,64 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
protected List<Document> createDocuments(String stringNodeRef, boolean isNew, boolean indexAllProperties,
boolean includeDirectoryDocuments)
{
AVMNodeDescriptor desc = avmService.lookup(-1, stringNodeRef);
List<Document> docs = new ArrayList<Document>();
if(desc == null)
if (stringNodeRef.startsWith("\u0000"))
{
Document idoc = new Document();
idoc.add(new Field("ID", stringNodeRef, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
docs.add(idoc);
return docs;
}
else if (stringNodeRef.startsWith(SNAP_SHOT_ID))
{
Document sdoc = new Document();
sdoc.add(new Field("ID", stringNodeRef, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
String storeName = stringNodeRef.substring(SNAP_SHOT_ID.length());
sdoc.add(new Field(SNAP_SHOT_ID, NumericEncoder.encode(avmService.getLatestSnapshotID(storeName)),
Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
sdoc.add(new Field(SNAP_SHOT_STORE, NumericEncoder.encode(avmService.getLatestSnapshotID(storeName)),
Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
docs.add(sdoc);
return docs;
}
AVMNodeDescriptor desc = avmService.lookup(endVersion, stringNodeRef);
if (desc == null)
{
return docs;
}
if (desc.isLayeredDirectory() || desc.isLayeredFile())
{
return docs;
}
List<Pair<Integer, String>> paths = avmService.getHeadPaths(desc);
}
List<Pair<Integer, String>> allPaths = avmService.getPaths(desc);
List<Pair<Integer, String>> paths = new ArrayList<Pair<Integer, String>> ();
for(Pair<Integer, String> pair: allPaths)
{
if(pair.getFirst().intValue() == endVersion)
{
paths.add(pair);
}
}
if(paths.size() == 0)
{
for(Pair<Integer, String> pair: allPaths)
{
if(pair.getFirst().intValue() == -1)
{
paths.add(pair);
}
}
}
if (paths.size() == 0)
{
return docs;
}
for (Pair<Integer, String> path : paths)
{
if(indexedPaths.contains(path.getSecond()))
if (indexedPaths.contains(path.getSecond()))
{
return docs;
}
@@ -297,37 +409,35 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
{
indexedPaths.add(path.getSecond());
}
AVMNode node = AVMDAOs.Instance().fAVMNodeDAO.getByID(desc.getId());
if (paths.size() > 0)
{
if (desc != null)
{
NodeRef nodeRef = AVMNodeConverter.ToNodeRef(-1, stringNodeRef);
NodeRef nodeRef = AVMNodeConverter.ToNodeRef(endVersion, stringNodeRef);
Document xdoc = new Document();
xdoc.add(new Field("ID", nodeRef.toString(), Field.Store.YES,
Field.Index.UN_TOKENIZED, Field.TermVector.NO));
xdoc.add(new Field("ID", nodeRef.toString(), Field.Store.YES, Field.Index.UN_TOKENIZED,
Field.TermVector.NO));
for (Pair<Integer, String> path : paths)
{
String[] splitPath = splitPath(path.getSecond());
@SuppressWarnings("unused")
String pathInStore = splitPath[1];
xdoc.add(new Field("ID", path.getSecond(),
Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
xdoc.add(new Field("ID", path.getSecond(), Field.Store.YES, Field.Index.UN_TOKENIZED,
Field.TermVector.NO));
}
xdoc.add(new Field("TX", AlfrescoTransactionSupport.getTransactionId(), Field.Store.YES,
Field.Index.UN_TOKENIZED, Field.TermVector.NO));
boolean isAtomic = true;
Map<QName, Serializable> properties = getIndexableProperties(desc, node, nodeRef, -1, stringNodeRef);
Map<QName, Serializable> properties = getIndexableProperties(desc, node, nodeRef, endVersion, stringNodeRef);
for (QName propertyName : properties.keySet())
{
Serializable value = properties.get(propertyName);
@@ -357,9 +467,9 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
String store = splitPath[0];
String pathInStore = splitPath[1];
SimplePath simplePath = new SimplePath(pathInStore);
StringBuilder xpathBuilder = new StringBuilder();
for(int i = 0; i < simplePath.size(); i++)
for (int i = 0; i < simplePath.size(); i++)
{
xpathBuilder.append("/{}").append(simplePath.get(i));
}
@@ -379,7 +489,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
boolean requiresSep = false;
for (int i = 0; i < simplePath.size() - 1; i++)
{
if(requiresSep)
if (requiresSep)
{
pathBuilder.append("/");
}
@@ -393,8 +503,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
qNameBuffer.append(ISO9075.getXPathName(QName.createQName("", simplePath
.get(simplePath.size() - 1))));
xdoc.add(new Field("PARENT",
ancestors.get(ancestors.size() - 1), Field.Store.YES,
xdoc.add(new Field("PARENT", ancestors.get(ancestors.size() - 1), Field.Store.YES,
Field.Index.UN_TOKENIZED, Field.TermVector.NO));
// TODO: Categories and LINKASPECT
@@ -407,20 +516,19 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
Document directoryEntry = new Document();
directoryEntry.add(new Field("ID", nodeRef.toString(), Field.Store.YES,
Field.Index.UN_TOKENIZED, Field.TermVector.NO));
directoryEntry.add(new Field("ID", path.getSecond(),
Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
directoryEntry.add(new Field("PATH", xpath, Field.Store.YES,
Field.Index.TOKENIZED, Field.TermVector.NO));
directoryEntry.add(new Field("ID", path.getSecond(), Field.Store.YES,
Field.Index.UN_TOKENIZED, Field.TermVector.NO));
directoryEntry.add(new Field("PATH", xpath, Field.Store.YES, Field.Index.TOKENIZED,
Field.TermVector.NO));
// Find all parent nodes.
for (String toAdd : ancestors)
{
directoryEntry.add(new Field("ANCESTOR", toAdd,
Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
directoryEntry.add(new Field("ANCESTOR", toAdd, Field.Store.NO,
Field.Index.UN_TOKENIZED, Field.TermVector.NO));
}
directoryEntry.add(new Field("ISCONTAINER", "T", Field.Store.YES,
Field.Index.UN_TOKENIZED, Field.TermVector.NO));
@@ -456,7 +564,7 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
xdoc.add(new Field("TYPE", ISO9075.getXPathName(typeQName), Field.Store.YES,
Field.Index.UN_TOKENIZED, Field.TermVector.NO));
for (QName classRef : avmService.getAspects(-1, stringNodeRef))
for (QName classRef : avmService.getAspects(endVersion, stringNodeRef))
{
xdoc.add(new Field("ASPECT", ISO9075.getXPathName(classRef), Field.Store.YES,
Field.Index.UN_TOKENIZED, Field.TermVector.NO));
@@ -534,11 +642,11 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
result.put(ContentModel.PROP_STORE_IDENTIFIER, nodeRef.getStoreRef().getIdentifier());
if (desc.isLayeredDirectory())
{
result.put(WCMModel.PROP_AVM_DIR_INDIRECTION, AVMNodeConverter.ToNodeRef(-1, desc.getIndirection()));
result.put(WCMModel.PROP_AVM_DIR_INDIRECTION, AVMNodeConverter.ToNodeRef(endVersion, desc.getIndirection()));
}
if (desc.isLayeredFile())
{
result.put(WCMModel.PROP_AVM_FILE_INDIRECTION, AVMNodeConverter.ToNodeRef(-1, desc.getIndirection()));
result.put(WCMModel.PROP_AVM_FILE_INDIRECTION, AVMNodeConverter.ToNodeRef(endVersion, desc.getIndirection()));
}
if (desc.isFile())
{
@@ -873,13 +981,32 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
@Override
protected void doCommit() throws IOException
{
if (indexUpdateStatus == IndexUpdateStatus.ASYNCHRONOUS)
{
setInfo(docs, getDeletions(), false);
// FTS does not trigger indexing request
}
else
{
setInfo(docs, getDeletions(), false);
// TODO: only register if required
fullTextSearchIndexer.requiresIndex(store);
}
if (callBack != null)
{
callBack.indexCompleted(store, remainingCount, null);
}
setInfo(docs, deletions, false);
}
@Override
protected void doRollBack() throws IOException
{
if (callBack != null)
{
callBack.indexCompleted(store, 0, null);
}
}
@Override
@@ -1014,29 +1141,68 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
}
public void deleteIndex(String store)
public void deleteIndex(String store, IndexMode mode)
{
checkAbleToDoWork(IndexUpdateStatus.SYNCRONOUS);
try
{
deleteAll();
switch (mode)
{
case ASYNCHRONOUS:
asyncronousDeleteIndex(store);
break;
case SYNCHRONOUS:
syncronousDeleteIndex(store);
break;
case UNINDEXED:
// nothing to do
break;
}
}
catch (LuceneIndexException e)
{
setRollbackOnly();
throw new LuceneIndexException("Delete index failed", e);
}
}
public void createIndex(String store)
public void syncronousDeleteIndex(String store)
{
if(s_logger.isDebugEnabled())
{
s_logger.debug("Sync delete for "+store);
}
deleteAll();
}
public void asyncronousDeleteIndex(String store)
{
if(s_logger.isDebugEnabled())
{
s_logger.debug("Async delete for "+store);
}
index("\u0000BG:DELETE:" + store+":"+GUID.generate());
fullTextSearchIndexer.requiresIndex(AVMNodeConverter.ToStoreRef(store));
}
public void createIndex(String store, IndexMode mode)
{
checkAbleToDoWork(IndexUpdateStatus.SYNCRONOUS);
try
{
@SuppressWarnings("unused")
AVMNodeDescriptor rootDesc = avmService.getStoreRoot(-1, store);
index(store+":/");
switch (mode)
{
case ASYNCHRONOUS:
asyncronousCreateIndex(store);
break;
case SYNCHRONOUS:
syncronousCreateIndex(store);
break;
case UNINDEXED:
// nothing to do
break;
}
}
catch (LuceneIndexException e)
{
@@ -1044,4 +1210,123 @@ public class AVMLuceneIndexerImpl extends AbstractLuceneIndexerImpl<String> impl
throw new LuceneIndexException("Create index failed", e);
}
}
public void syncronousCreateIndex(String store)
{
if(s_logger.isDebugEnabled())
{
s_logger.debug("Sync create for "+store);
}
@SuppressWarnings("unused")
AVMNodeDescriptor rootDesc = avmService.getStoreRoot(-1, store);
index(store + ":/");
}
public void asyncronousCreateIndex(String store)
{
if(s_logger.isDebugEnabled())
{
s_logger.debug("Async create for "+store);
}
index("\u0000BG:CREATE:" + store+":"+GUID.generate());
fullTextSearchIndexer.requiresIndex(AVMNodeConverter.ToStoreRef(store));
}
public void registerCallBack(FTSIndexerAware callBack)
{
this.callBack = callBack;
}
public int updateFullTextSearch(int size)
{
checkAbleToDoWork(IndexUpdateStatus.ASYNCHRONOUS);
try
{
PrefixQuery query = new PrefixQuery(new Term("ID", "\u0000BG:"));
String action = null;
Searcher searcher = null;
try
{
searcher = getSearcher(null);
// commit on another thread - appears like there is no index ...try later
if (searcher == null)
{
remainingCount = size;
return 0;
}
Hits hits;
try
{
hits = searcher.search(query);
}
catch (IOException e)
{
throw new LuceneIndexException(
"Failed to execute query to find content which needs updating in the index", e);
}
if (hits.length() > 0)
{
Document doc = hits.doc(0);
action = doc.getField("ID").stringValue();
String[] split = action.split(":");
if(split[1].equals("DELETE"))
{
deleteAll("\u0000BG:");
}
else if(split[1].equals("CREATE"))
{
syncronousCreateIndex(split[2]);
}
else if(split[1].equals("STORE"))
{
synchronousIndex(split[2], Integer.parseInt(split[3]), Integer.parseInt(split[4]));
}
deletions.add(action);
remainingCount = hits.length() - 1;
return 1;
}
else
{
remainingCount = 0;
return 0;
}
}
finally
{
if (searcher != null)
{
try
{
searcher.close();
}
catch (IOException e)
{
throw new LuceneIndexException("Failed to close searcher", e);
}
}
}
}
catch (IOException e)
{
setRollbackOnly();
throw new LuceneIndexException("Failed FTS update", e);
}
catch (LuceneIndexException e)
{
setRollbackOnly();
throw new LuceneIndexException("Failed FTS update", e);
}
}
public void setFullTextSearchIndexer(FullTextSearchIndexer fullTextSearchIndexer)
{
this.fullTextSearchIndexer = fullTextSearchIndexer;
}
}

52
source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerImpl.java
View File

@@ -63,15 +63,15 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
/**
* An index
*/
INDEX,
INDEX,
/**
* A reindex
*/
REINDEX,
REINDEX,
/**
* A delete
*/
DELETE,
DELETE,
/**
* A cascaded reindex (ensures directory structre is ok)
*/
@@ -83,13 +83,13 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
/**
* Inde is unchanged
*/
UNMODIFIED,
UNMODIFIED,
/**
* Index is being changein in TX
* Index is being changein in TX
*/
SYNCRONOUS,
SYNCRONOUS,
/**
* Index is eiong changed by a background upate
* Index is eiong changed by a background upate
*/
ASYNCHRONOUS;
}
@@ -379,7 +379,8 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
/**
* Commit this index
* @throws LuceneIndexException
*
* @throws LuceneIndexException
*/
public void commit() throws LuceneIndexException
{
@@ -441,7 +442,7 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
* serialisation against the index as would a data base transaction.
*
* @return the tx state
* @throws LuceneIndexException
* @throws LuceneIndexException
*/
public int prepare() throws LuceneIndexException
{
@@ -498,7 +499,8 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
/**
* Roll back the index changes (this just means they are never added)
* @throws LuceneIndexException
*
* @throws LuceneIndexException
*/
public void rollback() throws LuceneIndexException
{
@@ -819,6 +821,7 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
/**
* Are we deleting leaves only (not meta data)
*
* @return - deleting only nodes.
*/
public boolean getDeleteOnlyNodes()
@@ -828,6 +831,7 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
/**
* Get the deletions
*
* @return - the ids to delete
*/
public Set<String> getDeletions()
@@ -837,9 +841,18 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
/**
* Delete all entries from the index.
*
*/
public void deleteAll()
{
deleteAll(null);
}
/**
* Delete all index entries which do not start with the goven prefix
*
* @param prefix
*/
public void deleteAll(String prefix)
{
IndexReader mainReader = null;
try
@@ -851,7 +864,10 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
{
Document document = mainReader.document(doc);
String[] ids = document.getValues("ID");
deletions.add(ids[ids.length - 1]);
if ((prefix == null) || nonStartwWith(ids, prefix))
{
deletions.add(ids[ids.length - 1]);
}
}
}
@@ -877,4 +893,16 @@ public abstract class AbstractLuceneIndexerImpl<T> extends AbstractLuceneBase
}
}
private boolean nonStartwWith(String[] values, String prefix)
{
for (String value : values)
{
if (value.startsWith(prefix))
{
return false;
}
}
return true;
}
}