inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-07-24 17:32:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Lucene config and TX fixes

Browse Source 
git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@2293 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Andrew Hind
2006-02-03 13:25:42 +00:00
parent 29488351b1
commit f9ccb9ad25
5 changed files with 192 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
config/alfresco/core-services-context.xml
View File

@@ -246,6 +246,16 @@
<property name="indexerMaxFieldLength">
<value>${lucene.indexer.maxFieldLength}</value>
</property>
<property name="writeLockTimeout">
<value>${lucene.write.lock.timeout}</value>
</property>
<property name="commitLockTimeout">
<value>${lucene.commit.lock.timeout}</value>
</property>
<property name="lockPollInterval">
<value>${lucene.lock.poll.interval}</value>
</property>
</bean>
<!-- Indexer and searchers for lucene -->

9
config/alfresco/repository.properties
View File

@@ -42,6 +42,10 @@ lucene.indexer.maxMergeDocs=100000
#
lucene.indexer.maxFieldLength=10000
lucene.write.lock.timeout=10000
lucene.commit.lock.timeout=100000
lucene.lock.poll.interval=100
# Database configuration
db.driver=org.gjt.mm.mysql.Driver
@@ -50,6 +54,11 @@ db.url=jdbc:mysql:///${db.name}
db.username=alfresco
db.password=alfresco
#db.name=alfresco
#db.url=jdbc:mysql://qasi/${db.name}
#db.username=alfresco
#db.password=alfresco
# Email configuration
mail.host=

97
source/java/org/alfresco/repo/search/impl/lucene/LuceneBase.java
View File

@@ -35,12 +35,10 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
/**
* Common support for abstracting the lucene indexer from its configuration and
* management requirements.
* Common support for abstracting the lucene indexer from its configuration and management requirements.
*
* <p>
* This class defines where the indexes are stored. This should be via a
* configurable Bean property in Spring.
* This class defines where the indexes are stored. This should be via a configurable Bean property in Spring.
*
* <p>
* The default file structure is
@@ -51,13 +49,10 @@ import org.apache.lucene.store.FSDirectory;
* </ol>
*
* <p>
* The IndexWriter and IndexReader for a given index are toggled (one should be
* used for delete and the other for write). These are reused/closed/initialised
* as required.
* The IndexWriter and IndexReader for a given index are toggled (one should be used for delete and the other for write). These are reused/closed/initialised as required.
*
* <p>
* The index deltas are buffered to memory and persisted in the file system as
* required.
* The index deltas are buffered to memory and persisted in the file system as required.
*
* @author Andy Hind
*
@@ -86,15 +81,13 @@ public abstract class LuceneBase implements Lockable
private File undoDir;
/**
* The index reader for the on file delta. (This should no coexist with the
* writer)
* The index reader for the on file delta. (This should no coexist with the writer)
*/
private IndexReader deltaReader;
/**
* The writer for the delta to file. (This should no coexist with the
* reader)
* The writer for the delta to file. (This should no coexist with the reader)
*/
private IndexWriter deltaWriter;
@@ -134,8 +127,7 @@ public abstract class LuceneBase implements Lockable
// "lucene-indexes";
/**
* Initialise the configuration elements of the lucene store indexers and
* searchers.
* Initialise the configuration elements of the lucene store indexers and searchers.
*
* @param store
* @param deltaId
@@ -192,8 +184,7 @@ public abstract class LuceneBase implements Lockable
}
/**
* Utility method to find the path to the transactional store for this index
* delta
* Utility method to find the path to the transactional store for this index delta
*
* @return
*/
@@ -226,8 +217,7 @@ public abstract class LuceneBase implements Lockable
}
/**
* Utility method to initiliase a lucene FSDirectorya at a given location.
* We may try and delete the directory when the JVM exits.
* Utility method to initiliase a lucene FSDirectorya at a given location. We may try and delete the directory when the JVM exits.
*
* @param path
* @param temp
@@ -267,9 +257,7 @@ public abstract class LuceneBase implements Lockable
}
/**
* Get a searcher for the main index TODO: Split out support for the main
* index. We really only need this if we want to search over the changing
* index before it is committed
* Get a searcher for the main index TODO: Split out support for the main index. We really only need this if we want to search over the changing index before it is committed
*
* @return
* @throws IOException
@@ -511,8 +499,7 @@ public abstract class LuceneBase implements Lockable
}
/**
* Save the in memory delta to the disk, make sure there is nothing held in
* memory
* Save the in memory delta to the disk, make sure there is nothing held in memory
*
* @throws IOException
*/
@@ -527,9 +514,7 @@ public abstract class LuceneBase implements Lockable
/**
* Get all the locks so we can expect a merge to succeed
*
* The delta should be thread local so we do not have to worry about
* contentention TODO: Worry about main index contentention of readers and
* writers @
* The delta should be thread local so we do not have to worry about contentention TODO: Worry about main index contentention of readers and writers @
* @throws IOException
*/
protected void prepareToMergeIntoMain() throws LuceneIndexException
@@ -569,8 +554,7 @@ public abstract class LuceneBase implements Lockable
* Merge the delta in the main index. The delta still exists on disk.
*
* @param terms
* A list of terms that identifiy documents to be deleted from
* the main index before the delta os merged in.
* A list of terms that identifiy documents to be deleted from the main index before the delta os merged in.
*
* @throws IOException
*/
@@ -581,43 +565,44 @@ public abstract class LuceneBase implements Lockable
throw new LuceneIndexException("Must hold the write lock to merge");
}
if (!indexExists(baseDir))
try
{
if (s_logger.isDebugEnabled())
if (!indexExists(baseDir))
{
s_logger.debug("Creating base index " + baseDir);
if (s_logger.isDebugEnabled())
{
s_logger.debug("Creating base index " + baseDir);
}
try
{
mainWriter = new IndexWriter(baseDir, new LuceneAnalyser(dictionaryService), true);
mainWriter.setUseCompoundFile(true);
mainWriter.close();
if (s_logger.isDebugEnabled())
{
s_logger.debug("Created base index " + baseDir);
}
}
catch (IOException e)
{
s_logger.error("Error", e);
throw new LuceneIndexException("Failed to create empty base index at " + baseDir, e);
}
}
try
{
mainWriter = new IndexWriter(baseDir, new LuceneAnalyser(dictionaryService), true);
mainWriter.setUseCompoundFile(true);
mainWriter.close();
mainReader = IndexReader.open(baseDir);
if (s_logger.isDebugEnabled())
{
s_logger.debug("Created base index " + baseDir);
s_logger.debug("Opened base index for deletes " + baseDir);
}
}
catch (IOException e)
{
s_logger.error("Error", e);
throw new LuceneIndexException("Failed to create empty base index at " + baseDir, e);
throw new LuceneIndexException("Failed to create base index reader at " + baseDir, e);
}
}
try
{
mainReader = IndexReader.open(baseDir);
if (s_logger.isDebugEnabled())
{
s_logger.debug("Opened base index for deletes " + baseDir);
}
}
catch (IOException e)
{
s_logger.error("Error", e);
throw new LuceneIndexException("Failed to create base index reader at " + baseDir, e);
}
try
{
// Do the deletions
try
{
@@ -740,11 +725,9 @@ public abstract class LuceneBase implements Lockable
/**
* Delete the delta and make this instance unusable
*
* This tries to tidy up all it can. It is possible some stuff will remain
* if errors are throws else where
* This tries to tidy up all it can. It is possible some stuff will remain if errors are throws else where
*
* TODO: Support for cleaning up transactions - need to support recovery and
* knowing of we are prepared
* TODO: Support for cleaning up transactions - need to support recovery and knowing of we are prepared
*
*/
protected void deleteDelta() throws LuceneIndexException

17
source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerAndSearcherFactory.java
View File

@@ -50,7 +50,9 @@ import org.alfresco.util.GUID;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.store.Lock;
import org.quartz.Job;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
@@ -853,6 +855,21 @@ public class LuceneIndexerAndSearcherFactory implements LuceneIndexerAndSearcher
BooleanQuery.setMaxClauseCount(this.queryMaxClauses);
}
public void setWriteLockTimeout(long timeout)
{
IndexWriter.WRITE_LOCK_TIMEOUT = timeout;
}
public void setCommitLockTimeout(long timeout)
{
IndexWriter.COMMIT_LOCK_TIMEOUT = timeout;
}
public void setLockPollInterval(long time)
{
Lock.LOCK_POLL_INTERVAL = time;
}
public int getIndexerMaxFieldLength()
{
return indexerMaxFieldLength;

184
source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerImpl.java
View File

@@ -77,8 +77,7 @@ import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TermQuery;
/**
* The implementation of the lucene based indexer. Supports basic transactional
* behaviour if used on its own.
* The implementation of the lucene based indexer. Supports basic transactional behaviour if used on its own.
*
* @author andyh
*
@@ -112,8 +111,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
private ContentService contentService;
/**
* A list of all deletions we have made - at merge these deletions need to
* be made against the main index.
* A list of all deletions we have made - at merge these deletions need to be made against the main index.
*
* TODO: Consider if this information needs to be persisted for recovery
*/
@@ -133,9 +131,8 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
private boolean isModified = false;
/**
* Flag to indicte if we are doing an in transactional delta or a batch
* update to the index. If true, we are just fixing up non atomically
* indexed things from one or more other updates.
* Flag to indicte if we are doing an in transactional delta or a batch update to the index. If true, we are just fixing up non atomically indexed things from one or more other
* updates.
*/
private Boolean isFTSUpdate = null;
@@ -199,13 +196,12 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
this.contentService = contentService;
}
/***************************************************************************
/*******************************************************************************************************************************************************************************
* * Indexer Implementation * **************************
*/
/**
* Utility method to check we are in the correct state to do work Also keeps
* track of the dirty flag.
* Utility method to check we are in the correct state to do work Also keeps track of the dirty flag.
*
*/
@@ -557,92 +553,106 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
try
{
mainReader = getReader();
deltaReader = getDeltaReader();
mainSearcher = new IndexSearcher(mainReader);
deltaSearcher = new IndexSearcher(deltaReader);
for (Helper helper : toFTSIndex)
try
{
BooleanQuery query = new BooleanQuery();
query.add(new TermQuery(new Term("ID", helper.nodeRef.toString())), true, false);
query.add(new TermQuery(new Term("TX", helper.tx)), true, false);
query.add(new TermQuery(new Term("ISNODE", "T")), false, false);
mainReader = getReader();
deltaReader = getDeltaReader();
mainSearcher = new IndexSearcher(mainReader);
deltaSearcher = new IndexSearcher(deltaReader);
try
for (Helper helper : toFTSIndex)
{
Hits hits = mainSearcher.search(query);
if (hits.length() > 0)
BooleanQuery query = new BooleanQuery();
query.add(new TermQuery(new Term("ID", helper.nodeRef.toString())), true, false);
query.add(new TermQuery(new Term("TX", helper.tx)), true, false);
query.add(new TermQuery(new Term("ISNODE", "T")), false, false);
try
{
// No change
for (int i = 0; i < hits.length(); i++)
Hits hits = mainSearcher.search(query);
if (hits.length() > 0)
{
mainReader.delete(hits.id(i));
// No change
for (int i = 0; i < hits.length(); i++)
{
mainReader.delete(hits.id(i));
}
}
else
{
hits = deltaSearcher.search(query);
for (int i = 0; i < hits.length(); i++)
{
deltaReader.delete(hits.id(i));
}
}
}
else
catch (IOException e)
{
hits = deltaSearcher.search(query);
for (int i = 0; i < hits.length(); i++)
{
deltaReader.delete(hits.id(i));
}
throw new LuceneIndexException("Failed to delete an FTS update from the original index", e);
}
}
catch (IOException e)
}
finally
{
if (deltaSearcher != null)
{
throw new LuceneIndexException("Failed to delete an FTS update from the original index", e);
try
{
deltaSearcher.close();
}
catch (IOException e)
{
s_logger.warn("Failed to close delta searcher", e);
}
}
if (mainSearcher != null)
{
try
{
mainSearcher.close();
}
catch (IOException e)
{
s_logger.warn("Failed to close main searcher", e);
}
}
try
{
closeDeltaReader();
}
catch (LuceneIndexException e)
{
s_logger.warn("Failed to close delta reader", e);
}
if (mainReader != null)
{
try
{
mainReader.close();
}
catch (IOException e)
{
s_logger.warn("Failed to close main reader", e);
}
}
}
mergeDeltaIntoMain(new LinkedHashSet<Term>());
}
catch (LuceneIndexException e)
{
// If anything goes wrong we try and do a roll back
rollback();
throw new LuceneIndexException("Commit failed", e);
}
finally
{
if (deltaSearcher != null)
{
try
{
deltaSearcher.close();
}
catch (IOException e)
{
s_logger.warn("Failed to close delta searcher", e);
}
}
if (mainSearcher != null)
{
try
{
mainSearcher.close();
}
catch (IOException e)
{
s_logger.warn("Failed to close main searcher", e);
}
}
try
{
closeDeltaReader();
}
catch (LuceneIndexException e)
{
s_logger.warn("Failed to close delta reader", e);
}
if (mainReader != null)
{
try
{
mainReader.close();
}
catch (IOException e)
{
s_logger.warn("Failed to close main reader", e);
}
}
// Make sure we tidy up
deleteDelta();
}
mergeDeltaIntoMain(new LinkedHashSet<Term>());
}
/**
@@ -650,8 +660,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
*
* At the moment this makes sure we have all the locks
*
* TODO: This is not doing proper serialisation against the index as would a
* data base transaction.
* TODO: This is not doing proper serialisation against the index as would a data base transaction.
*
* @return
*/
@@ -749,8 +758,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
}
/**
* Mark this index for roll back only. This action can not be reversed. It
* will reject all other work and only allow roll back.
* Mark this index for roll back only. This action can not be reversed. It will reject all other work and only allow roll back.
*
*/
@@ -1072,8 +1080,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
try
{
writer.addDocument(doc /*
* TODO: Select the language based
* analyser
* TODO: Select the language based analyser
*/);
}
catch (IOException e)
@@ -1396,7 +1403,8 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
if (index)
{
// store mimetype in index - even if content does not index it is useful
doc.add(new Field(attributeName+".mimetype", contentData.getMimetype(), false, true, false));
doc.add(new Field(attributeName + ".mimetype", contentData.getMimetype(), false, true,
false));
ContentReader reader = contentService.getReader(nodeRef, propertyName);
if (reader != null && reader.exists())
@@ -1454,7 +1462,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
InputStream ris = reader.getContentInputStream();
try
{
isr = new InputStreamReader(ris,"UTF-8");
isr = new InputStreamReader(ris, "UTF-8");
}
catch (UnsupportedEncodingException e)
{
@@ -1465,7 +1473,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
ris = reader.getReader().getContentInputStream();
try
{
isr = new InputStreamReader(ris,"UTF-8");
isr = new InputStreamReader(ris, "UTF-8");
}
catch (UnsupportedEncodingException e)
{
@@ -1726,9 +1734,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer
try
{
writer.addDocument(doc /*
* TODO: Select the
* language based
* analyser
* TODO: Select the language based analyser
*/);
}
catch (IOException e)