diff --git a/config/alfresco/core-services-context.xml b/config/alfresco/core-services-context.xml index d485395ea2..6f2257b476 100644 --- a/config/alfresco/core-services-context.xml +++ b/config/alfresco/core-services-context.xml @@ -246,6 +246,16 @@ ${lucene.indexer.maxFieldLength} + + ${lucene.write.lock.timeout} + + + ${lucene.commit.lock.timeout} + + + ${lucene.lock.poll.interval} + + diff --git a/config/alfresco/repository.properties b/config/alfresco/repository.properties index 70b09948ea..21651022ae 100644 --- a/config/alfresco/repository.properties +++ b/config/alfresco/repository.properties @@ -42,6 +42,10 @@ lucene.indexer.maxMergeDocs=100000 # lucene.indexer.maxFieldLength=10000 +lucene.write.lock.timeout=10000 +lucene.commit.lock.timeout=100000 +lucene.lock.poll.interval=100 + # Database configuration db.driver=org.gjt.mm.mysql.Driver @@ -50,6 +54,11 @@ db.url=jdbc:mysql:///${db.name} db.username=alfresco db.password=alfresco +#db.name=alfresco +#db.url=jdbc:mysql://qasi/${db.name} +#db.username=alfresco +#db.password=alfresco + # Email configuration mail.host= diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneBase.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneBase.java index bd85d76239..0b2c870fb9 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/LuceneBase.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneBase.java @@ -35,12 +35,10 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; /** - * Common support for abstracting the lucene indexer from its configuration and - * management requirements. + * Common support for abstracting the lucene indexer from its configuration and management requirements. * *

- * This class defines where the indexes are stored. This should be via a - * configurable Bean property in Spring. + * This class defines where the indexes are stored. This should be via a configurable Bean property in Spring. * *

* The default file structure is @@ -51,13 +49,10 @@ import org.apache.lucene.store.FSDirectory; * * *

- * The IndexWriter and IndexReader for a given index are toggled (one should be - * used for delete and the other for write). These are reused/closed/initialised - * as required. + * The IndexWriter and IndexReader for a given index are toggled (one should be used for delete and the other for write). These are reused/closed/initialised as required. * *

- * The index deltas are buffered to memory and persisted in the file system as - * required. + * The index deltas are buffered to memory and persisted in the file system as required. * * @author Andy Hind * @@ -86,15 +81,13 @@ public abstract class LuceneBase implements Lockable private File undoDir; /** - * The index reader for the on file delta. (This should no coexist with the - * writer) + * The index reader for the on file delta. (This should no coexist with the writer) */ private IndexReader deltaReader; /** - * The writer for the delta to file. (This should no coexist with the - * reader) + * The writer for the delta to file. (This should no coexist with the reader) */ private IndexWriter deltaWriter; @@ -134,8 +127,7 @@ public abstract class LuceneBase implements Lockable // "lucene-indexes"; /** - * Initialise the configuration elements of the lucene store indexers and - * searchers. + * Initialise the configuration elements of the lucene store indexers and searchers. * * @param store * @param deltaId @@ -192,8 +184,7 @@ public abstract class LuceneBase implements Lockable } /** - * Utility method to find the path to the transactional store for this index - * delta + * Utility method to find the path to the transactional store for this index delta * * @return */ @@ -226,8 +217,7 @@ public abstract class LuceneBase implements Lockable } /** - * Utility method to initiliase a lucene FSDirectorya at a given location. - * We may try and delete the directory when the JVM exits. + * Utility method to initiliase a lucene FSDirectorya at a given location. We may try and delete the directory when the JVM exits. * * @param path * @param temp @@ -267,9 +257,7 @@ public abstract class LuceneBase implements Lockable } /** - * Get a searcher for the main index TODO: Split out support for the main - * index. We really only need this if we want to search over the changing - * index before it is committed + * Get a searcher for the main index TODO: Split out support for the main index. We really only need this if we want to search over the changing index before it is committed * * @return * @throws IOException @@ -511,8 +499,7 @@ public abstract class LuceneBase implements Lockable } /** - * Save the in memory delta to the disk, make sure there is nothing held in - * memory + * Save the in memory delta to the disk, make sure there is nothing held in memory * * @throws IOException */ @@ -527,9 +514,7 @@ public abstract class LuceneBase implements Lockable /** * Get all the locks so we can expect a merge to succeed * - * The delta should be thread local so we do not have to worry about - * contentention TODO: Worry about main index contentention of readers and - * writers @ + * The delta should be thread local so we do not have to worry about contentention TODO: Worry about main index contentention of readers and writers @ * @throws IOException */ protected void prepareToMergeIntoMain() throws LuceneIndexException @@ -569,8 +554,7 @@ public abstract class LuceneBase implements Lockable * Merge the delta in the main index. The delta still exists on disk. * * @param terms - * A list of terms that identifiy documents to be deleted from - * the main index before the delta os merged in. + * A list of terms that identifiy documents to be deleted from the main index before the delta os merged in. * * @throws IOException */ @@ -581,43 +565,44 @@ public abstract class LuceneBase implements Lockable throw new LuceneIndexException("Must hold the write lock to merge"); } - if (!indexExists(baseDir)) + try { - if (s_logger.isDebugEnabled()) + if (!indexExists(baseDir)) { - s_logger.debug("Creating base index " + baseDir); + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Creating base index " + baseDir); + } + try + { + mainWriter = new IndexWriter(baseDir, new LuceneAnalyser(dictionaryService), true); + mainWriter.setUseCompoundFile(true); + mainWriter.close(); + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Created base index " + baseDir); + } + } + catch (IOException e) + { + s_logger.error("Error", e); + throw new LuceneIndexException("Failed to create empty base index at " + baseDir, e); + } } try { - mainWriter = new IndexWriter(baseDir, new LuceneAnalyser(dictionaryService), true); - mainWriter.setUseCompoundFile(true); - mainWriter.close(); + mainReader = IndexReader.open(baseDir); if (s_logger.isDebugEnabled()) { - s_logger.debug("Created base index " + baseDir); + s_logger.debug("Opened base index for deletes " + baseDir); } } catch (IOException e) { s_logger.error("Error", e); - throw new LuceneIndexException("Failed to create empty base index at " + baseDir, e); + throw new LuceneIndexException("Failed to create base index reader at " + baseDir, e); } - } - try - { - mainReader = IndexReader.open(baseDir); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Opened base index for deletes " + baseDir); - } - } - catch (IOException e) - { - s_logger.error("Error", e); - throw new LuceneIndexException("Failed to create base index reader at " + baseDir, e); - } - try - { + // Do the deletions try { @@ -740,11 +725,9 @@ public abstract class LuceneBase implements Lockable /** * Delete the delta and make this instance unusable * - * This tries to tidy up all it can. It is possible some stuff will remain - * if errors are throws else where + * This tries to tidy up all it can. It is possible some stuff will remain if errors are throws else where * - * TODO: Support for cleaning up transactions - need to support recovery and - * knowing of we are prepared + * TODO: Support for cleaning up transactions - need to support recovery and knowing of we are prepared * */ protected void deleteDelta() throws LuceneIndexException diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerAndSearcherFactory.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerAndSearcherFactory.java index 1baca6491d..00b3026795 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerAndSearcherFactory.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerAndSearcherFactory.java @@ -50,7 +50,9 @@ import org.alfresco.util.GUID; import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.store.Lock; import org.quartz.Job; import org.quartz.JobDataMap; import org.quartz.JobExecutionContext; @@ -853,6 +855,21 @@ public class LuceneIndexerAndSearcherFactory implements LuceneIndexerAndSearcher BooleanQuery.setMaxClauseCount(this.queryMaxClauses); } + public void setWriteLockTimeout(long timeout) + { + IndexWriter.WRITE_LOCK_TIMEOUT = timeout; + } + + public void setCommitLockTimeout(long timeout) + { + IndexWriter.COMMIT_LOCK_TIMEOUT = timeout; + } + + public void setLockPollInterval(long time) + { + Lock.LOCK_POLL_INTERVAL = time; + } + public int getIndexerMaxFieldLength() { return indexerMaxFieldLength; diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerImpl.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerImpl.java index acebaa3cfc..afdb3b7038 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerImpl.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneIndexerImpl.java @@ -77,11 +77,10 @@ import org.apache.lucene.search.Searcher; import org.apache.lucene.search.TermQuery; /** - * The implementation of the lucene based indexer. Supports basic transactional - * behaviour if used on its own. + * The implementation of the lucene based indexer. Supports basic transactional behaviour if used on its own. * * @author andyh - * + * */ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer { @@ -112,8 +111,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer private ContentService contentService; /** - * A list of all deletions we have made - at merge these deletions need to - * be made against the main index. + * A list of all deletions we have made - at merge these deletions need to be made against the main index. * * TODO: Consider if this information needs to be persisted for recovery */ @@ -133,9 +131,8 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer private boolean isModified = false; /** - * Flag to indicte if we are doing an in transactional delta or a batch - * update to the index. If true, we are just fixing up non atomically - * indexed things from one or more other updates. + * Flag to indicte if we are doing an in transactional delta or a batch update to the index. If true, we are just fixing up non atomically indexed things from one or more other + * updates. */ private Boolean isFTSUpdate = null; @@ -199,13 +196,12 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer this.contentService = contentService; } - /*************************************************************************** + /******************************************************************************************************************************************************************************* * * Indexer Implementation * ************************** */ /** - * Utility method to check we are in the correct state to do work Also keeps - * track of the dirty flag. + * Utility method to check we are in the correct state to do work Also keeps track of the dirty flag. * */ @@ -557,92 +553,106 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer try { - mainReader = getReader(); - deltaReader = getDeltaReader(); - mainSearcher = new IndexSearcher(mainReader); - deltaSearcher = new IndexSearcher(deltaReader); - - for (Helper helper : toFTSIndex) + try { - BooleanQuery query = new BooleanQuery(); - query.add(new TermQuery(new Term("ID", helper.nodeRef.toString())), true, false); - query.add(new TermQuery(new Term("TX", helper.tx)), true, false); - query.add(new TermQuery(new Term("ISNODE", "T")), false, false); + mainReader = getReader(); + deltaReader = getDeltaReader(); + mainSearcher = new IndexSearcher(mainReader); + deltaSearcher = new IndexSearcher(deltaReader); - try + for (Helper helper : toFTSIndex) { - Hits hits = mainSearcher.search(query); - if (hits.length() > 0) + BooleanQuery query = new BooleanQuery(); + query.add(new TermQuery(new Term("ID", helper.nodeRef.toString())), true, false); + query.add(new TermQuery(new Term("TX", helper.tx)), true, false); + query.add(new TermQuery(new Term("ISNODE", "T")), false, false); + + try { - // No change - for (int i = 0; i < hits.length(); i++) + Hits hits = mainSearcher.search(query); + if (hits.length() > 0) { - mainReader.delete(hits.id(i)); + // No change + for (int i = 0; i < hits.length(); i++) + { + mainReader.delete(hits.id(i)); + } + } + else + { + hits = deltaSearcher.search(query); + for (int i = 0; i < hits.length(); i++) + { + deltaReader.delete(hits.id(i)); + } } } - else + catch (IOException e) { - hits = deltaSearcher.search(query); - for (int i = 0; i < hits.length(); i++) - { - deltaReader.delete(hits.id(i)); - } + throw new LuceneIndexException("Failed to delete an FTS update from the original index", e); } } - catch (IOException e) + + } + finally + { + if (deltaSearcher != null) { - throw new LuceneIndexException("Failed to delete an FTS update from the original index", e); + try + { + deltaSearcher.close(); + } + catch (IOException e) + { + s_logger.warn("Failed to close delta searcher", e); + } + } + if (mainSearcher != null) + { + try + { + mainSearcher.close(); + } + catch (IOException e) + { + s_logger.warn("Failed to close main searcher", e); + } + } + try + { + closeDeltaReader(); + } + catch (LuceneIndexException e) + { + s_logger.warn("Failed to close delta reader", e); + } + if (mainReader != null) + { + try + { + mainReader.close(); + } + catch (IOException e) + { + s_logger.warn("Failed to close main reader", e); + } } } + mergeDeltaIntoMain(new LinkedHashSet()); + } + catch (LuceneIndexException e) + { + // If anything goes wrong we try and do a roll back + rollback(); + throw new LuceneIndexException("Commit failed", e); } finally { - if (deltaSearcher != null) - { - try - { - deltaSearcher.close(); - } - catch (IOException e) - { - s_logger.warn("Failed to close delta searcher", e); - } - } - if (mainSearcher != null) - { - try - { - mainSearcher.close(); - } - catch (IOException e) - { - s_logger.warn("Failed to close main searcher", e); - } - } - try - { - closeDeltaReader(); - } - catch (LuceneIndexException e) - { - s_logger.warn("Failed to close delta reader", e); - } - if (mainReader != null) - { - try - { - mainReader.close(); - } - catch (IOException e) - { - s_logger.warn("Failed to close main reader", e); - } - } + // Make sure we tidy up + deleteDelta(); } - mergeDeltaIntoMain(new LinkedHashSet()); - } /** @@ -650,8 +660,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer * * At the moment this makes sure we have all the locks * - * TODO: This is not doing proper serialisation against the index as would a - * data base transaction. + * TODO: This is not doing proper serialisation against the index as would a data base transaction. * * @return */ @@ -749,8 +758,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer } /** - * Mark this index for roll back only. This action can not be reversed. It - * will reject all other work and only allow roll back. + * Mark this index for roll back only. This action can not be reversed. It will reject all other work and only allow roll back. * */ @@ -1072,8 +1080,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer try { writer.addDocument(doc /* - * TODO: Select the language based - * analyser + * TODO: Select the language based analyser */); } catch (IOException e) @@ -1396,8 +1403,9 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer if (index) { // store mimetype in index - even if content does not index it is useful - doc.add(new Field(attributeName+".mimetype", contentData.getMimetype(), false, true, false)); - + doc.add(new Field(attributeName + ".mimetype", contentData.getMimetype(), false, true, + false)); + ContentReader reader = contentService.getReader(nodeRef, propertyName); if (reader != null && reader.exists()) { @@ -1450,28 +1458,28 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer // reader, but only if the reader is valid if (readerReady) { - InputStreamReader isr = null; - InputStream ris = reader.getContentInputStream(); - try - { - isr = new InputStreamReader(ris,"UTF-8"); - } - catch (UnsupportedEncodingException e) - { - isr = new InputStreamReader(ris); - } + InputStreamReader isr = null; + InputStream ris = reader.getContentInputStream(); + try + { + isr = new InputStreamReader(ris, "UTF-8"); + } + catch (UnsupportedEncodingException e) + { + isr = new InputStreamReader(ris); + } doc.add(Field.Text("TEXT", isr)); - - ris = reader.getReader().getContentInputStream(); - try - { - isr = new InputStreamReader(ris,"UTF-8"); - } - catch (UnsupportedEncodingException e) - { - isr = new InputStreamReader(ris); - } - + + ris = reader.getReader().getContentInputStream(); + try + { + isr = new InputStreamReader(ris, "UTF-8"); + } + catch (UnsupportedEncodingException e) + { + isr = new InputStreamReader(ris); + } + doc.add(Field.Text("@" + QName.createQName(propertyName.getNamespaceURI(), ISO9075 .encode(propertyName.getLocalName())), isr)); @@ -1726,9 +1734,7 @@ public class LuceneIndexerImpl extends LuceneBase implements LuceneIndexer try { writer.addDocument(doc /* - * TODO: Select the - * language based - * analyser + * TODO: Select the language based analyser */); } catch (IOException e)