diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java index 56c8942ce8..180d4a2468 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java @@ -26,36 +26,42 @@ class IndexEntry /** * The type of the index entry */ - IndexType type; + private IndexType type; /** * The unique name of the index entry */ - String name; + private String name; /** * The preceeding index name. * Allows deltas etc to apply to the index or an overlay for example. */ - String parentName; + private String parentName; /** - * The status of the inedx entry + * The status of the index entry */ - TransactionStatus status; + private TransactionStatus status; /** * If merging, the id where the result is going */ - String mergeId; - - IndexEntry(IndexType type, String name, String parentName, TransactionStatus status, String mergeId) + private String mergeId; + + private long documentCount; + + private long deletions; + + IndexEntry(IndexType type, String name, String parentName, TransactionStatus status, String mergeId, long documentCount, long deletions) { this.type = type; this.name = name; this.parentName = parentName; this.status = status; this.mergeId = mergeId; + this.documentCount = documentCount; + this.deletions = deletions; } public String getMergeId() @@ -107,6 +113,36 @@ class IndexEntry { this.type = type; } + + public long getDocumentCount() + { + return documentCount; + } + + public void setDocumentCount(long documentCount) + { + this.documentCount = documentCount; + } + + public long getDeletions() + { + return deletions; + } + + public void setDeletions(long deletions) + { + this.deletions = deletions; + } + + public String toString() + { + StringBuilder builder = new StringBuilder(); + builder.append(" Name=").append(getName()).append(" "); + builder.append("Type=").append(getType()).append(" "); + builder.append("Status=").append(getStatus()).append(" "); + builder.append("Docs=").append(getDocumentCount()).append(" "); + return builder.toString(); + } } \ No newline at end of file diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java index cf9c5c69c9..e86cc1ca5b 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java @@ -16,8 +16,14 @@ */ package org.alfresco.repo.search.impl.lucene.index; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.io.UnsupportedEncodingException; @@ -27,29 +33,36 @@ import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.FileChannel.MapMode; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.zip.CRC32; -import javax.swing.plaf.multi.MultiInternalFrameUI; - import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.repo.search.IndexerException; -import org.alfresco.repo.search.impl.lucene.ClosingIndexSearcher; import org.alfresco.repo.search.impl.lucene.FilterIndexReaderByNodeRefs; -import org.alfresco.repo.search.impl.lucene.LuceneIndexer; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.util.GUID; +import org.apache.log4j.Logger; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.hibernate.criterion.Order; /** * The information that makes up an index. @@ -73,7 +86,7 @@ import org.apache.lucene.index.Term; * Incomplete delete merging does not matter - the overlay would still exist and be treated as such. So a document may be deleted in the index as well as in the applied overlay. It * is still correctly deleted. * - * NOTE: Public methods locak as required, the private methods assume that the appropriate locks have been obtained. + * NOTE: Public methods lock as required, the private methods assume that the appropriate locks have been obtained. * * TODO: Write element status into individual directories. This would be enough for recovery if both index files are lost or corrupted. * @@ -83,6 +96,7 @@ import org.apache.lucene.index.Term; */ public class IndexInfo { + private static Logger s_logger = Logger.getLogger(IndexInfo.class); private static final boolean useNIOMemoryMapping = true; @@ -96,10 +110,12 @@ public class IndexInfo */ private static String INDEX_INFO_BACKUP = "IndexInfoBackup"; + private static String INDEX_INFO_DELETIONS = "IndexInfoDeletions"; + /** * Is this index shared by more than one repository? We can make many lock optimisations if the index is not shared. */ - private boolean indexIsShared; + private boolean indexIsShared = false; /** * The directory that holds the index @@ -155,12 +171,12 @@ public class IndexInfo /** * Index writers for deltas */ - private HashMap indexWriters = new HashMap(); + private Map indexWriters = Collections.synchronizedMap(new HashMap()); /** * Index Readers for deltas */ - private HashMap indexReaders = new HashMap(); + private Map indexReaders = Collections.synchronizedMap(new HashMap()); /** * Map of state transitions @@ -168,6 +184,18 @@ public class IndexInfo private EnumMap transitions = new EnumMap( TransactionStatus.class); + private ConcurrentLinkedQueue deleteQueue = new ConcurrentLinkedQueue(); + + private Cleaner cleaner = new Cleaner(); + + private Thread cleanerThread; + + private Merger merger = new Merger(); + + private Thread mergerThread; + + private Directory emptyIndex = new RAMDirectory(); + /** * Construct an index in the given directory. * @@ -212,7 +240,7 @@ public class IndexInfo // Read info from disk if this is not a new index. if (version == -1) { - readWriteLock.writeLock().lock(); + getWriteLock(); try { doWithFileLock(new LockWork() @@ -220,6 +248,70 @@ public class IndexInfo public Object doWork() throws Exception { setStatusFromFile(); + + if (!indexIsShared) + { + HashSet deletable = new HashSet(); + // clean up + for (IndexEntry entry : indexEntries.values()) + { + switch (entry.getStatus()) + { + case ACTIVE: + case MARKED_ROLLBACK: + case NO_TRANSACTION: + case PREPARING: + case ROLLEDBACK: + case ROLLINGBACK: + case MERGE_TARGET: + case UNKNOWN: + case PREPARED: + case DELETABLE: + if (s_logger.isInfoEnabled()) + { + s_logger.info("Deleting index entry " + entry); + } + entry.setStatus(TransactionStatus.DELETABLE); + deletable.add(entry.getName()); + break; + case COMMITTED_DELETING: + case MERGE: + if (s_logger.isInfoEnabled()) + { + s_logger.info("Resetting merge to committed " + entry); + } + entry.setStatus(TransactionStatus.COMMITTED); + break; + case COMMITTING: + // do the commit + if (s_logger.isInfoEnabled()) + { + s_logger.info("Committing " + entry); + } + entry.setStatus(TransactionStatus.COMMITTED); + mainIndexReader = null; + break; + case COMMITTED: + default: + // nothing to do + break; + } + } + for (String id : deletable) + { + IndexEntry entry = indexEntries.remove(id); + deleteQueue.add(id); + } + synchronized (cleaner) + { + cleaner.notify(); + } + synchronized (merger) + { + merger.notify(); + } + writeStatus(); + } return null; } @@ -227,9 +319,34 @@ public class IndexInfo } finally { - readWriteLock.writeLock().unlock(); + releaseWriteLock(); } } + // TODO: Add unrecognised folders for deletion. + cleanerThread = new Thread(cleaner); + cleanerThread.setDaemon(true); + cleanerThread.setName("Index cleaner thread"); + cleanerThread.start(); + + mergerThread = new Thread(merger); + mergerThread.setDaemon(true); + mergerThread.setName("Index merger thread"); + mergerThread.start(); + + IndexWriter writer; + try + { + writer = new IndexWriter(emptyIndex, new StandardAnalyzer(), true); + writer.setUseCompoundFile(true); + writer.minMergeDocs = 1000; + writer.mergeFactor = 5; + writer.maxMergeDocs = 1000000; + } + catch (IOException e) + { + throw new IndexerException("Failed to create an empty in memory index!"); + } + } /** @@ -241,13 +358,24 @@ public class IndexInfo */ public IndexReader getDeltaIndexReader(String id) throws IOException { + // No read lock required as the delta should be bound to one thread only + // Index readers are simply thread safe IndexReader reader = indexReaders.get(id); if (reader == null) { // close index writer if required closeDeltaIndexWriter(id); - File location = ensureDeltaExistsAndIsRegistered(id); - reader = IndexReader.open(location); + // Check the index knows about the transaction + File location = ensureDeltaIsRegistered(id); + // Create a dummy index reader to deal with empty indexes and not persist these. + if (IndexReader.indexExists(location)) + { + reader = IndexReader.open(location); + } + else + { + reader = IndexReader.open(emptyIndex); + } indexReaders.put(id, reader); } return reader; @@ -260,54 +388,76 @@ public class IndexInfo * @return * @throws IOException */ - private File ensureDeltaExistsAndIsRegistered(String id) throws IOException + private File ensureDeltaIsRegistered(String id) throws IOException { + // A write lock is required if we have to update the local index entries. + // There should only be one thread trying to access this delta. File location = new File(indexDirectory, id); - if (!IndexReader.indexExists(location)) - { - IndexWriter creator = new IndexWriter(location, new StandardAnalyzer(), true); - creator.setUseCompoundFile(true); - creator.close(); - } - readWriteLock.readLock().lock(); + getReadLock(); try { if (!indexEntries.containsKey(id)) { - readWriteLock.readLock().unlock(); + releaseReadLock(); // release to upgrade to write lock - readWriteLock.writeLock().lock(); + getWriteLock(); try { + // Make sure the index exists if (!indexEntries.containsKey(id)) { - indexEntries.put(id, new IndexEntry(IndexType.DELTA, id, "", TransactionStatus.ACTIVE, "")); + indexEntries.put(id, + new IndexEntry(IndexType.DELTA, id, "", TransactionStatus.ACTIVE, "", 0, 0)); } } finally { - // Downgrade - readWriteLock.readLock().lock(); - readWriteLock.writeLock().unlock(); + // Downgrade lock + getReadLock(); + releaseWriteLock(); } } } finally { - readWriteLock.readLock().unlock(); + // Release the lock + releaseReadLock(); } return location; } + private IndexWriter makeDeltaIndexWriter(File location) throws IOException + { + if (!IndexReader.indexExists(location)) + { + IndexWriter creator = new IndexWriter(location, new StandardAnalyzer(), true); + creator.setUseCompoundFile(true); + creator.minMergeDocs = 1000; + creator.mergeFactor = 5; + creator.maxMergeDocs = 1000000; + return creator; + } + return null; + } + public IndexWriter getDeltaIndexWriter(String id, Analyzer analyzer) throws IOException { + // No read lock required as the delta should be bound to one thread only IndexWriter writer = indexWriters.get(id); if (writer == null) { // close index writer if required closeDeltaIndexReader(id); - File location = ensureDeltaExistsAndIsRegistered(id); - writer = new IndexWriter(location, analyzer, false); + File location = ensureDeltaIsRegistered(id); + writer = makeDeltaIndexWriter(location); + if (writer == null) + { + writer = new IndexWriter(location, analyzer, false); + writer.setUseCompoundFile(true); + writer.minMergeDocs = 1000; + writer.mergeFactor = 5; + writer.maxMergeDocs = 1000000; + } indexWriters.put(id, writer); } return writer; @@ -315,6 +465,7 @@ public class IndexInfo public void closeDeltaIndexReader(String id) throws IOException { + // No lock required as the delta applied to one thread. The delta is still active. IndexReader reader = indexReaders.get(id); if (reader != null) { @@ -325,6 +476,7 @@ public class IndexInfo public void closeDeltaIndexWriter(String id) throws IOException { + // No lock required as the delta applied to one thread. The delta is still active. IndexWriter writer = indexWriters.get(id); if (writer != null) { @@ -333,15 +485,102 @@ public class IndexInfo } } - public IndexReader getMainIndexReferenceCountingReadOnlyIndexReader() throws IOException + public void closeDelta(String id) throws IOException { - readWriteLock.readLock().lock(); + closeDeltaIndexReader(id); + closeDeltaIndexWriter(id); + } + + public Set getDeletions(String id) throws IOException + { + // Check state + Set deletions = new HashSet(); + File location = new File(indexDirectory, id); + File file = new File(location, INDEX_INFO_DELETIONS); + if (!file.exists()) + { + return Collections. emptySet(); + } + DataInputStream is = new DataInputStream(new BufferedInputStream(new FileInputStream(file))); + int size = is.readInt(); + for (int i = 0; i < size; i++) + { + String ref = is.readUTF(); + deletions.add(new NodeRef(ref)); + } + is.close(); + return deletions; + + } + + public void setPreparedState(String id, Set toDelete, long documents) throws IOException + { + // Check state + if (toDelete.size() > 0) + { + File location = new File(indexDirectory, id); + if (!location.exists()) + { + if (!location.mkdirs()) + { + throw new IndexerException("Failed to make index directory " + location); + } + } + DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(new File(location, + INDEX_INFO_DELETIONS)))); + os.writeInt(toDelete.size()); + for (NodeRef ref : toDelete) + { + os.writeUTF(ref.toString()); + } + os.flush(); + os.close(); + } + getWriteLock(); try { + IndexEntry entry = indexEntries.get(id); + if (entry == null) + { + throw new IndexerException("Invalid index delta id " + id); + } + if (entry.getStatus() != TransactionStatus.PREPARING) + { + throw new IndexerException("Deletes and doc count can only be set on a preparing index"); + } + entry.setDocumentCount(documents); + entry.setDeletions(toDelete.size()); + } + finally + { + releaseWriteLock(); + } + } + + public IndexReader getMainIndexReferenceCountingReadOnlyIndexReader() throws IOException + { + getReadLock(); + try + { + if (indexIsShared && !checkVersion()) + { + releaseReadLock(); + getWriteLock(); + try + { + mainIndexReader = null; + } + finally + { + getReadLock(); + releaseWriteLock(); + } + } + if (mainIndexReader == null) { - readWriteLock.readLock().unlock(); - readWriteLock.writeLock().lock(); + releaseReadLock(); + getWriteLock(); try { if (mainIndexReader == null) @@ -356,31 +595,54 @@ public class IndexInfo }); mainIndexReader = createMainIndexReader(); + } } finally { - readWriteLock.readLock(); - readWriteLock.writeLock().unlock(); + getReadLock(); + releaseWriteLock(); } } + ReferenceCounting refCount = (ReferenceCounting) mainIndexReader; + refCount.incrementReferenceCount(); + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Main index reader references = " + refCount.getReferenceCount()); + } return mainIndexReader; } finally { - readWriteLock.readLock().unlock(); + releaseReadLock(); } } - public IndexReader getMainIndexReferenceCountingReadOnlyIndexReader(LuceneIndexer luceneIndexer) throws IOException + public IndexReader getMainIndexReferenceCountingReadOnlyIndexReader(String id, Set deletions) + throws IOException { - readWriteLock.readLock().lock(); + getReadLock(); try { + if (indexIsShared && !checkVersion()) + { + releaseReadLock(); + getWriteLock(); + try + { + mainIndexReader = null; + } + finally + { + getReadLock(); + releaseWriteLock(); + } + } + if (mainIndexReader == null) { - readWriteLock.readLock().unlock(); - readWriteLock.writeLock().lock(); + releaseReadLock(); + getWriteLock(); try { if (mainIndexReader == null) @@ -395,27 +657,34 @@ public class IndexInfo }); mainIndexReader = createMainIndexReader(); + } } finally { - readWriteLock.readLock(); - readWriteLock.writeLock().unlock(); + getReadLock(); + releaseWriteLock(); } } + ReferenceCounting refCount = (ReferenceCounting) mainIndexReader; + refCount.incrementReferenceCount(); + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Main index reader references = " + refCount.getReferenceCount()); + } // Combine the index delta with the main index // Make sure the index is written to disk // TODO: Should use the in memory index but we often end up forcing to disk anyway. // Is it worth it? - luceneIndexer.flushPending(); - IndexReader deltaReader = getDeltaIndexReader(luceneIndexer.getDeltaId()); + // luceneIndexer.flushPending(); + IndexReader deltaReader = getDeltaIndexReader(id); IndexReader reader = new MultiReader(new IndexReader[] { - new FilterIndexReaderByNodeRefs(mainIndexReader, luceneIndexer.getDeletions()), deltaReader }); + new FilterIndexReaderByNodeRefs(mainIndexReader, deletions), deltaReader }); return reader; } finally { - readWriteLock.readLock().unlock(); + releaseReadLock(); } } @@ -423,22 +692,49 @@ public class IndexInfo throws IOException { final Transition transition = getTransition(state); - readWriteLock.writeLock().lock(); + getWriteLock(); try { - doWithFileLock(new LockWork() + if (transition.requiresFileLock()) { - public Object doWork() throws Exception + doWithFileLock(new LockWork() { - transition.transition(id, toDelete, read); - return null; - } + public Object doWork() throws Exception + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Start Index " + id + " state = " + state); + } + dumpInfo(); + transition.transition(id, toDelete, read); + if (s_logger.isDebugEnabled()) + { + s_logger.debug("End Index " + id + " state = " + state); + } + dumpInfo(); + return null; + } - }); + }); + } + else + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Start Index " + id + " state = " + state); + } + dumpInfo(); + transition.transition(id, toDelete, read); + if (s_logger.isDebugEnabled()) + { + s_logger.debug("End Index " + id + " state = " + state); + } + dumpInfo(); + } } finally { - readWriteLock.writeLock().unlock(); + releaseWriteLock(); } } @@ -469,12 +765,15 @@ public class IndexInfo transitions.put(TransactionStatus.COMMITTED, new CommittedTransition()); transitions.put(TransactionStatus.ROLLINGBACK, new RollingBackTransition()); transitions.put(TransactionStatus.ROLLEDBACK, new RolledBackTransition()); - + transitions.put(TransactionStatus.DELETABLE, new DeletableTransition()); + transitions.put(TransactionStatus.ACTIVE, new ActiveTransition()); } private interface Transition { void transition(String id, Set toDelete, Set read) throws IOException; + + boolean requiresFileLock(); } private class PreparingTransition implements Transition @@ -490,7 +789,6 @@ public class IndexInfo if (TransactionStatus.PREPARING.follows(entry.getStatus())) { entry.setStatus(TransactionStatus.PREPARING); - writeStatus(); } else { @@ -498,6 +796,11 @@ public class IndexInfo + id + " from " + entry.getStatus() + " to " + TransactionStatus.PREPARING); } } + + public boolean requiresFileLock() + { + return !TransactionStatus.PREPARING.isTransient(); + } } private class PreparedTransition implements Transition @@ -512,8 +815,45 @@ public class IndexInfo if (TransactionStatus.PREPARED.follows(entry.getStatus())) { + if ((entry.getDeletions() + entry.getDocumentCount()) > 0) + { + LinkedHashMap reordered = new LinkedHashMap(); + boolean addedPreparedEntry = false; + for (IndexEntry current : indexEntries.values()) + { + if (!current.getStatus().canBeReordered()) + { + reordered.put(current.getName(), current); + } + else if (!addedPreparedEntry) + { + reordered.put(entry.getName(), entry); + reordered.put(current.getName(), current); + addedPreparedEntry = true; + } + else if (current.getName().equals(entry.getName())) + { + // skip as we are moving it + } + else + { + reordered.put(current.getName(), current); + } + } + + if (indexEntries.size() != reordered.size()) + { + indexEntries = reordered; + dumpInfo(); + throw new IndexerException("Concurrent modification error"); + } + indexEntries = reordered; + } entry.setStatus(TransactionStatus.PREPARED); - writeStatus(); + if ((entry.getDeletions() + entry.getDocumentCount()) > 0) + { + writeStatus(); + } } else { @@ -521,6 +861,11 @@ public class IndexInfo + id + " from " + entry.getStatus() + " to " + TransactionStatus.PREPARED); } } + + public boolean requiresFileLock() + { + return !TransactionStatus.PREPARED.isTransient(); + } } private class CommittingTransition implements Transition @@ -536,7 +881,6 @@ public class IndexInfo if (TransactionStatus.COMMITTING.follows(entry.getStatus())) { entry.setStatus(TransactionStatus.COMMITTING); - writeStatus(); } else { @@ -544,6 +888,11 @@ public class IndexInfo + id + " from " + entry.getStatus() + " to " + TransactionStatus.COMMITTING); } } + + public boolean requiresFileLock() + { + return !TransactionStatus.COMMITTING.isTransient(); + } } private class CommittedTransition implements Transition @@ -559,9 +908,31 @@ public class IndexInfo if (TransactionStatus.COMMITTED.follows(entry.getStatus())) { // Do the deletions + if ((entry.getDocumentCount() + entry.getDeletions()) == 0) + { + indexEntries.remove(id); + } + else + { + entry.setStatus(TransactionStatus.COMMITTED); + // TODO: optimise to index for no deletions + // have to allow for this in the application of deletions, + writeStatus(); + if (mainIndexReader != null) + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("... invalidating main index reader"); + } + ((ReferenceCounting) mainIndexReader).setInvalidForReuse(); + mainIndexReader = null; + } - entry.setStatus(TransactionStatus.COMMITTED); - writeStatus(); + synchronized (merger) + { + merger.notify(); + } + } } else { @@ -569,6 +940,11 @@ public class IndexInfo + id + " from " + entry.getStatus() + " to " + TransactionStatus.COMMITTED); } } + + public boolean requiresFileLock() + { + return !TransactionStatus.COMMITTED.isTransient(); + } } private class RollingBackTransition implements Transition @@ -592,6 +968,11 @@ public class IndexInfo + id + " from " + entry.getStatus() + " to " + TransactionStatus.ROLLINGBACK); } } + + public boolean requiresFileLock() + { + return !TransactionStatus.ROLLINGBACK.isTransient(); + } } private class RolledBackTransition implements Transition @@ -615,6 +996,71 @@ public class IndexInfo + id + " from " + entry.getStatus() + " to " + TransactionStatus.ROLLEDBACK); } } + + public boolean requiresFileLock() + { + return !TransactionStatus.ROLLEDBACK.isTransient(); + } + } + + private class DeletableTransition implements Transition + { + public void transition(String id, Set toDelete, Set read) throws IOException + { + IndexEntry entry = indexEntries.get(id); + if (entry == null) + { + throw new IndexerException("Unknown transaction " + id); + } + + if (TransactionStatus.DELETABLE.follows(entry.getStatus())) + { + indexEntries.remove(id); + deleteQueue.add(id); + synchronized (cleaner) + { + cleaner.notify(); + } + writeStatus(); + } + else + { + throw new IndexerException("Invalid transition for " + + id + " from " + entry.getStatus() + " to " + TransactionStatus.DELETABLE); + } + } + + public boolean requiresFileLock() + { + return !TransactionStatus.DELETABLE.isTransient(); + } + } + + private class ActiveTransition implements Transition + { + public void transition(String id, Set toDelete, Set read) throws IOException + { + IndexEntry entry = indexEntries.get(id); + if (entry != null) + { + throw new IndexerException("TX Already active " + id); + } + + if (TransactionStatus.ACTIVE.follows(null)) + { + indexEntries.put(id, new IndexEntry(IndexType.DELTA, id, "", TransactionStatus.ACTIVE, "", 0, 0)); + } + else + { + throw new IndexerException("Invalid transition for " + + id + " from " + entry.getStatus() + " to " + TransactionStatus.ACTIVE); + } + } + + public boolean requiresFileLock() + { + return !TransactionStatus.ACTIVE.isTransient(); + } } // @@ -709,12 +1155,24 @@ public class IndexInfo for (String id : inValid) { IndexReader reader = referenceCountingReadOnlyIndexReaders.remove(id); + if (s_logger.isDebugEnabled()) + { + s_logger.debug("... invalidating sub reader " + id); + } ReferenceCounting referenceCounting = (ReferenceCounting) reader; referenceCounting.setInvalidForReuse(); hasInvalid = true; } if (hasInvalid) { + if (mainIndexReader != null) + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("... invalidating main index reader"); + } + ((ReferenceCounting) mainIndexReader).setInvalidForReuse(); + } mainIndexReader = null; } } @@ -734,10 +1192,23 @@ public class IndexInfo } else { - reader = new MultiReader(new IndexReader[] { reader, subReader }); + if (entry.getType() == IndexType.INDEX) + { + reader = new MultiReader(new IndexReader[] { reader, subReader }); + } + else if (entry.getType() == IndexType.DELTA) + { + reader = new MultiReader(new IndexReader[] { + new FilterIndexReaderByNodeRefs(reader, getDeletions(entry.getName())), subReader }); + } } } } + if (reader == null) + { + reader = IndexReader.open(emptyIndex); + } + reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader("MainReader", reader); return reader; } @@ -747,9 +1218,19 @@ public class IndexInfo if (reader == null) { File location = new File(indexDirectory, id); - reader = IndexReader.open(location); - reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(reader); + if (IndexReader.indexExists(location)) + { + reader = IndexReader.open(location); + } + else + { + reader = IndexReader.open(emptyIndex); + } + reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(id, reader); + referenceCountingReadOnlyIndexReaders.put(id, reader); } + ReferenceCounting referenceCounting = (ReferenceCounting) reader; + referenceCounting.incrementReferenceCount(); return reader; } @@ -762,7 +1243,14 @@ public class IndexInfo catch (IOException e) { // The first data file is corrupt so we fall back to the back up - return checkVersion(indexInfoBackupChannel); + try + { + return checkVersion(indexInfoBackupChannel); + } + catch (IOException ee) + { + return false; + } } } @@ -823,6 +1311,8 @@ public class IndexInfo int size = buffer.getInt(); crc32.update(size); LinkedHashMap newIndexEntries = new LinkedHashMap(); + // Not all state is saved some is specific to this index so we need to add the transient stuff. + // Until things are committed they are not shared unless it is prepared for (int i = 0; i < size; i++) { String indexTypeString = readString(buffer, crc32); @@ -853,11 +1343,30 @@ public class IndexInfo String mergeId = readString(buffer, crc32); - newIndexEntries.put(name, new IndexEntry(indexType, name, parentName, status, mergeId)); + long documentCount = buffer.getLong(); + crc32.update((int) (documentCount >>> 32) & 0xFFFFFFFF); + crc32.update((int) (documentCount >>> 0) & 0xFFFFFFFF); + + long deletions = buffer.getLong(); + crc32.update((int) (deletions >>> 32) & 0xFFFFFFFF); + crc32.update((int) (deletions >>> 0) & 0xFFFFFFFF); + + if (!status.isTransient()) + { + newIndexEntries.put(name, new IndexEntry(indexType, name, parentName, status, mergeId, + documentCount, deletions)); + } } long onDiskCRC32 = buffer.getLong(); if (crc32.getValue() == onDiskCRC32) { + for (IndexEntry entry : indexEntries.values()) + { + if (entry.getStatus().isTransient()) + { + newIndexEntries.put(entry.getName(), entry); + } + } version = onDiskVersion; indexEntries = newIndexEntries; } @@ -873,27 +1382,32 @@ public class IndexInfo private String readString(ByteBuffer buffer, CRC32 crc32) throws UnsupportedEncodingException { int size = buffer.getInt(); + byte[] bytes = new byte[size]; + buffer.get(bytes); char[] chars = new char[size]; for (int i = 0; i < size; i++) { - chars[i] = buffer.getChar(); + chars[i] = (char) bytes[i]; } - String string = new String(chars); - - crc32.update(string.getBytes("UTF-8")); - return string; + crc32.update(bytes); + return new String(chars); } private void writeString(ByteBuffer buffer, CRC32 crc32, String string) throws UnsupportedEncodingException { char[] chars = string.toCharArray(); - buffer.putInt(chars.length); - + byte[] bytes = new byte[chars.length]; for (int i = 0; i < chars.length; i++) { - buffer.putChar(chars[i]); + if (chars[i] > 0xFF) + { + throw new UnsupportedEncodingException(); + } + bytes[i] = (byte) chars[i]; } - crc32.update(string.getBytes("UTF-8")); + buffer.putInt(bytes.length); + buffer.put(bytes); + crc32.update(bytes); } private void writeStatus() throws IOException @@ -943,6 +1457,14 @@ public class IndexInfo writeString(buffer, crc32, entryStatus); writeString(buffer, crc32, entry.getMergeId()); + + buffer.putLong(entry.getDocumentCount()); + crc32.update((int) (entry.getDocumentCount() >>> 32) & 0xFFFFFFFF); + crc32.update((int) (entry.getDocumentCount() >>> 0) & 0xFFFFFFFF); + + buffer.putLong(entry.getDeletions()); + crc32.update((int) (entry.getDeletions() >>> 32) & 0xFFFFFFFF); + crc32.update((int) (entry.getDeletions() >>> 0) & 0xFFFFFFFF); } buffer.putLong(crc32.getValue()); @@ -958,7 +1480,7 @@ public class IndexInfo } } - private long getBufferSize() + private long getBufferSize() throws IOException { long size = 0; size += 8; @@ -966,12 +1488,14 @@ public class IndexInfo for (IndexEntry entry : indexEntries.values()) { String entryType = entry.getType().toString(); - size += (entryType.length() * 2) + 4; - size += (entry.getName().length() * 2) + 4; - size += (entry.getParentName().length() * 2) + 4; + size += (entryType.length()) + 4; + size += (entry.getName().length()) + 4; + size += (entry.getParentName().length()) + 4; String entryStatus = entry.getStatus().toString(); - size += (entryStatus.length() * 2) + 4; - size += (entry.getMergeId().length() * 2) + 4; + size += (entryStatus.length()) + 4; + size += (entry.getMergeId().length()) + 4; + size += 8; + size += 8; } size += 8; return size; @@ -994,7 +1518,6 @@ public class IndexInfo if (!checkVersion()) { setStatusFromFile(); - clearOldReaders(); } } result = lockWork.doWork(); @@ -1032,7 +1555,14 @@ public class IndexInfo { System.setProperty("disableLuceneLocks", "true"); + HashSet deletions = new HashSet(); + for (int i = 0; i < 0; i++) + { + deletions.add(new NodeRef(new StoreRef("woof", "bingle"), GUID.generate())); + } + int repeat = 100; + int docs = 1; final IndexInfo ii = new IndexInfo(new File("c:\\indexTest")); long totalTimeA = 0; @@ -1041,17 +1571,33 @@ public class IndexInfo while (true) { long start = System.nanoTime(); - ii.indexEntries.clear(); - for (int i = 0; i < 100; i++) + for (int i = 0; i < repeat; i++) { String guid = GUID.generate(); - ii.indexEntries.put(guid, new IndexEntry(IndexType.DELTA, guid, GUID.generate(), - TransactionStatus.ACTIVE, "")); - ii.getDeltaIndexReader(guid); + ii.setStatus(guid, TransactionStatus.ACTIVE, null, null); + IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer()); + + for (int j = 0; j < docs; j++) + { + Document doc = new Document(); + for (int k = 0; k < 15; k++) + { + doc.add(new Field("ID" + k, guid + " " + j + " " + k, false, true, false)); + } + writer.addDocument(doc); + } + + ii.closeDeltaIndexWriter(guid); ii.setStatus(guid, TransactionStatus.PREPARING, null, null); + ii.setPreparedState(guid, deletions, docs); + ii.getDeletions(guid); ii.setStatus(guid, TransactionStatus.PREPARED, null, null); ii.setStatus(guid, TransactionStatus.COMMITTING, null, null); ii.setStatus(guid, TransactionStatus.COMMITTED, null, null); + for (int j = 0; j < 0; j++) + { + ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + } } long end = System.nanoTime(); @@ -1064,4 +1610,787 @@ public class IndexInfo + repeat + " in " + ((end - start) / 1000000000.0) + " average = " + average); } } + + /** + * Clean up support. + * + * @author Andy Hind + */ + private class Cleaner implements Runnable + { + + public void run() + { + boolean runnable = true; + while (runnable) + { + String id = null; + while ((id = deleteQueue.poll()) != null) + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Expunging " + id + " remaining " + deleteQueue.size()); + } + // try and delete + File location = new File(indexDirectory, id); + if (!deleteDirectory(location)) + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("DELETE FAILED"); + } + // try again later + deleteQueue.add(id); + } + } + synchronized (this) + { + try + { + // wait for more deletes + this.wait(); + } + catch (InterruptedException e) + { + runnable = false; + } + } + } + + } + + private boolean deleteDirectory(File file) + { + File[] children = file.listFiles(); + if (children != null) + { + for (int i = 0; i < children.length; i++) + { + File child = children[i]; + if (child.isDirectory()) + { + deleteDirectory(child); + } + else + { + if (child.exists() && !child.delete() && child.exists()) + { + return false; + } + } + } + } + if (file.exists() && !file.delete() && file.exists()) + { + return false; + } + return true; + } + + } + + /** + * Supported by one thread. + * + * 1) If the first index is a delta we can just change it to an index. + * + * There is now here to apply the deletions + * + * 2) Merge indexes + * + * Combine indexes together according to the target index merge strategy. This is a trade off to make an optimised index but not spend too much time merging and optimising + * small merges. + * + * 3) Apply next deletion set to indexes + * + * Apply the deletions for the first delta to all the other indexes. Deletes can be applied with relative impunity. If any are applied they take effect as required. + * + * 1) 2) and 3) are mutually exclusive try in order + * + * This could be supported in another thread + * + * 4) Merge deltas + * + * Merge two index deltas together. Starting at the end. Several merges can be going on at once. + * + * a) Find merge b) Set state c) apply deletions to the previous delta d) update state e) add deletions to the previous delta deletion list f) update state + * + */ + + private enum MergeAction + { + NONE, MERGE_INDEX, APPLY_DELTA_DELETION, MERGE_DELTA + } + + private class Merger implements Runnable + { + public void run() + { + boolean running = true; + + while (running) + { + // Get the read local to decide what to do + // Single JVM to start with + MergeAction action = MergeAction.NONE; + + getReadLock(); + try + { + if (indexIsShared && !checkVersion()) + { + releaseReadLock(); + getWriteLock(); + try + { + // Sync with disk image if required + doWithFileLock(new LockWork() + { + public Object doWork() throws Exception + { + return null; + } + }); + } + finally + { + getReadLock(); + releaseWriteLock(); + } + } + + int indexes = 0; + boolean mergingIndexes = false; + int deltas = 0; + boolean applyingDeletions = false; + + for (IndexEntry entry : indexEntries.values()) + { + if (entry.getType() == IndexType.INDEX) + { + indexes++; + if (entry.getStatus() == TransactionStatus.MERGE) + { + mergingIndexes = true; + } + } + else if (entry.getType() == IndexType.DELTA) + { + if (entry.getStatus() == TransactionStatus.COMMITTED) + { + deltas++; + } + if (entry.getStatus() == TransactionStatus.COMMITTED_DELETING) + { + applyingDeletions = true; + } + } + } + + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Indexes = " + indexes); + s_logger.debug("Merging = " + mergingIndexes); + s_logger.debug("Deltas = " + deltas); + s_logger.debug("Deleting = " + applyingDeletions); + } + + if (!mergingIndexes && !applyingDeletions) + { + + if ((indexes > 5) || (deltas > 5)) + { + if (indexes > deltas) + { + // Try merge + action = MergeAction.MERGE_INDEX; + } + else + { + // Try delete + action = MergeAction.APPLY_DELTA_DELETION; + + } + } + } + } + + catch (IOException e) + { + e.printStackTrace(); + // Ignore IO error and retry + } + finally + { + releaseReadLock(); + } + + if (action == MergeAction.APPLY_DELTA_DELETION) + { + mergeDeletions(); + } + else if (action == MergeAction.MERGE_INDEX) + { + mergeIndexes(); + } + + synchronized (this) + { + try + { + this.wait(); + } + catch (InterruptedException e) + { + running = false; + } + } + } + + } + + void mergeDeletions() + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Deleting ..."); + } + + // lock for deletions + final LinkedHashMap toDelete; + + getWriteLock(); + try + { + toDelete = doWithFileLock(new LockWork>() + { + public LinkedHashMap doWork() throws Exception + { + LinkedHashMap set = new LinkedHashMap(); + + for (IndexEntry entry : indexEntries.values()) + { + if ((entry.getType() == IndexType.INDEX) && (entry.getStatus() == TransactionStatus.MERGE)) + { + return set; + } + if ((entry.getType() == IndexType.DELTA) + && (entry.getStatus() == TransactionStatus.COMMITTED_DELETING)) + { + return set; + } + } + // Check it is not deleting + boolean foundDelta; + for (IndexEntry entry : indexEntries.values()) + { + if (entry.getType() == IndexType.DELTA) + { + if (entry.getStatus() == TransactionStatus.COMMITTED) + { + entry.setStatus(TransactionStatus.COMMITTED_DELETING); + set.put(entry.getName(), entry); + } + else if (entry.getStatus() == TransactionStatus.PREPARED) + { + break; + } + } + } + if (set.size() > 0) + { + writeStatus(); + } + return set; + + } + + }); + } + finally + { + getReadLock(); + releaseWriteLock(); + } + + LinkedHashMap indexes = new LinkedHashMap(); + try + { + for (IndexEntry entry : indexEntries.values()) + { + if (entry.getStatus() == TransactionStatus.COMMITTED_DELETING) + { + break; + } + indexes.put(entry.getName(), entry); + } + } + finally + { + releaseReadLock(); + } + + // Build readers + + boolean fail = false; + + final HashSet invalidIndexes = new HashSet(); + + final HashMap newIndexCounts = new HashMap(); + + try + { + LinkedHashMap readers = new LinkedHashMap(); + for (IndexEntry entry : indexes.values()) + { + File location = new File(indexDirectory, entry.getName()); + IndexReader reader; + if (IndexReader.indexExists(location)) + { + reader = IndexReader.open(location); + } + else + { + reader = IndexReader.open(emptyIndex); + } + readers.put(entry.getName(), reader); + } + + for (IndexEntry currentDelete : toDelete.values()) + { + Set deletions = getDeletions(currentDelete.getName()); + for (String key : readers.keySet()) + { + IndexReader reader = readers.get(key); + for (NodeRef nodeRef : deletions) + { + if (reader.delete(new Term("ID", nodeRef.toString())) > 0) + { + invalidIndexes.add(key); + } + } + + } + File location = new File(indexDirectory, currentDelete.getName()); + IndexReader reader; + if (IndexReader.indexExists(location)) + { + reader = IndexReader.open(location); + } + else + { + reader = IndexReader.open(emptyIndex); + } + readers.put(currentDelete.getName(), reader); + } + + for (String key : readers.keySet()) + { + IndexReader reader = readers.get(key); + // TODO:Set the new document count + newIndexCounts.put(key, new Long(reader.numDocs())); + reader.close(); + } + } + catch (IOException e) + { + e.printStackTrace(); + fail = true; + } + + final boolean wasDeleted = !fail; + getWriteLock(); + try + { + doWithFileLock(new LockWork() + { + public Object doWork() throws Exception + { + for (IndexEntry entry : toDelete.values()) + { + entry.setStatus(TransactionStatus.COMMITTED); + if (wasDeleted) + { + entry.setType(IndexType.INDEX); + entry.setDeletions(0); + } + + } + + for(String key : newIndexCounts.keySet()) + { + Long newCount = newIndexCounts.get(key); + IndexEntry entry = indexEntries.get(key); + entry.setDocumentCount(newCount); + } + + writeStatus(); + + for (String id : invalidIndexes) + { + IndexReader reader = referenceCountingReadOnlyIndexReaders.remove(id); + if (reader != null) + { + ReferenceCounting referenceCounting = (ReferenceCounting) reader; + referenceCounting.setInvalidForReuse(); + if (s_logger.isDebugEnabled()) + { + s_logger.debug("... invalidating sub reader after merge" + id); + } + } + } + if (invalidIndexes.size() > 0) + { + if (mainIndexReader != null) + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("... invalidating main index reader after merge"); + } + ((ReferenceCounting) mainIndexReader).setInvalidForReuse(); + } + mainIndexReader = null; + } + + if (s_logger.isDebugEnabled()) + { + for (String id : toDelete.keySet()) + { + s_logger.debug("...applied deletion for " + id); + } + for (String id : invalidIndexes) + { + s_logger.debug("...invalidated index " + id); + } + s_logger.debug("...deleting done"); + } + + dumpInfo(); + + return null; + } + + + }); + } + finally + { + releaseWriteLock(); + } + + + + // TODO: Flush readers etc + + } + + void mergeIndexes() + { + + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Merging..."); + } + + final LinkedHashMap toMerge; + + getWriteLock(); + try + { + toMerge = doWithFileLock(new LockWork>() + { + public LinkedHashMap doWork() throws Exception + { + LinkedHashMap set = new LinkedHashMap(); + + for (IndexEntry entry : indexEntries.values()) + { + if ((entry.getType() == IndexType.INDEX) && (entry.getStatus() == TransactionStatus.MERGE)) + { + return set; + } + if ((entry.getType() == IndexType.DELTA) + && (entry.getStatus() == TransactionStatus.COMMITTED_DELETING)) + { + return set; + } + } + + ArrayList mergeList = new ArrayList(); + for (IndexEntry entry : indexEntries.values()) + { + if ((entry.getType() == IndexType.INDEX) + && (entry.getStatus() == TransactionStatus.COMMITTED)) + { + mergeList.add(entry); + } + } + + int position = findMergeIndex(1, 1000000, 5, mergeList); + String firstMergeId = mergeList.get(position).getName(); + + long count = 0; + String guid = null; + if (position >= 0) + { + guid = GUID.generate(); + for (int i = position; i < mergeList.size(); i++) + { + IndexEntry entry = mergeList.get(i); + count += entry.getDocumentCount(); + set.put(entry.getName(), entry); + entry.setStatus(TransactionStatus.MERGE); + entry.setMergeId(guid); + } + } + + if (set.size() > 0) + { + IndexEntry target = new IndexEntry(IndexType.INDEX, guid, "", + TransactionStatus.MERGE_TARGET, guid, count, 0); + set.put(guid, target); + // rebuild merged index elements + LinkedHashMap reordered = new LinkedHashMap(); + for (IndexEntry current : indexEntries.values()) + { + if (current.getName().equals(firstMergeId)) + { + reordered.put(target.getName(), target); + } + reordered.put(current.getName(), current); + } + indexEntries = reordered; + writeStatus(); + } + return set; + + } + + }); + } + finally + { + releaseWriteLock(); + } + + if (s_logger.isDebugEnabled()) + { + s_logger.debug("....Merging..." + (toMerge.size() - 1)); + } + + boolean fail = false; + + try + { + if (toMerge.size() > 0) + { + int count = 0; + IndexReader[] readers = new IndexReader[toMerge.size() - 1]; + IndexWriter writer = null; + for (IndexEntry entry : toMerge.values()) + { + File location = new File(indexDirectory, entry.getName()); + if (entry.getStatus() == TransactionStatus.MERGE) + { + IndexReader reader; + if (IndexReader.indexExists(location)) + { + reader = IndexReader.open(location); + } + else + { + reader = IndexReader.open(emptyIndex); + } + readers[count++] = reader; + } + else if (entry.getStatus() == TransactionStatus.MERGE_TARGET) + { + writer = new IndexWriter(location, new StandardAnalyzer(), true); + writer.setUseCompoundFile(true); + writer.minMergeDocs = 1000; + writer.mergeFactor = 5; + writer.maxMergeDocs = 1000000; + } + } + writer.addIndexes(readers); + writer.close(); + for (IndexReader reader : readers) + { + reader.close(); + } + } + } + catch (IOException e) + { + e.printStackTrace(); + fail = true; + } + + final boolean wasMerged = !fail; + getWriteLock(); + try + { + doWithFileLock(new LockWork() + { + public Object doWork() throws Exception + { + HashSet toDelete = new HashSet(); + for (IndexEntry entry : toMerge.values()) + { + if (entry.getStatus() == TransactionStatus.MERGE) + { + if (wasMerged) + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("... deleting as merged " + entry.getName()); + } + toDelete.add(entry.getName()); + } + else + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("... committing as merge failed " + entry.getName()); + } + entry.setStatus(TransactionStatus.COMMITTED); + } + } + else if (entry.getStatus() == TransactionStatus.MERGE_TARGET) + { + if (wasMerged) + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("... committing merge target " + entry.getName()); + } + entry.setStatus(TransactionStatus.COMMITTED); + } + else + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("... deleting merge target as merge failed " + entry.getName()); + } + toDelete.add(entry.getName()); + } + } + } + for (String id : toDelete) + { + indexEntries.remove(id); + deleteQueue.add(id); + } + synchronized (cleaner) + { + cleaner.notify(); + } + + dumpInfo(); + + writeStatus(); + + clearOldReaders(); + + return null; + } + + }); + } + finally + { + releaseWriteLock(); + } + + if (s_logger.isDebugEnabled()) + { + s_logger.debug("..done merging"); + } + + } + + private final int findMergeIndex(long min, long max, int factor, List entries) throws IOException + { + // TODO: Support max + if (entries.size() <= factor) + { + return -1; + } + + int total = 0; + for (int i = factor; i < entries.size(); i++) + { + total += entries.get(i).getDocumentCount(); + } + + for (int i = factor - 1; i > 0; i--) + { + total += entries.get(i).getDocumentCount(); + if (total < entries.get(i - 1).getDocumentCount()) + { + return i; + } + } + return 0; + } + } + + private void dumpInfo() + { + readWriteLock.writeLock().lock(); + try + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug(""); + s_logger.debug("Entry List"); + for (IndexEntry entry : indexEntries.values()) + { + s_logger.debug(" " + entry.toString()); + } + } + } + finally + { + readWriteLock.writeLock().unlock(); + } + + } + + private void getWriteLock() + { + readWriteLock.writeLock().lock(); + if (s_logger.isDebugEnabled()) + { + s_logger.debug("GOT WRITE LOCK - " + Thread.currentThread().getName()); + } + } + + private void releaseWriteLock() + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("RELEASES WRITE LOCK - " + Thread.currentThread().getName()); + } + readWriteLock.writeLock().unlock(); + } + + private void getReadLock() + { + readWriteLock.readLock().lock(); + if (s_logger.isDebugEnabled()) + { + s_logger.debug("GOT READ LOCK - " + Thread.currentThread().getName()); + } + } + + private void releaseReadLock() + { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("RELEASES READ LOCK - " + Thread.currentThread().getName()); + } + readWriteLock.readLock().unlock(); + } + } diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfoTest.java b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfoTest.java new file mode 100644 index 0000000000..e834c21c83 --- /dev/null +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfoTest.java @@ -0,0 +1,871 @@ +/* + * Copyright (C) 2005 Alfresco, Inc. + * + * Licensed under the Mozilla Public License version 1.1 + * with a permitted attribution clause. You may obtain a + * copy of the License at + * + * http://www.alfresco.org/legal/license.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.alfresco.repo.search.impl.lucene.index; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; + +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.util.GUID; +import org.alfresco.util.TempFileProvider; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; + +import junit.framework.TestCase; + +public class IndexInfoTest extends TestCase +{ + + public static final String[] WORD_LIST = { "aardvark", "banana", "cucumber", "daffodil", "emu", "frog", "gibbon", + "humour", "injection", "jelly", "key", "lemur", "monkey", "number", "open", "plummet", "quest", + "replication", "steam", "tunnel", "uncommon", "verbose", "where", "xylem", "yellow", "zebra", "alpha", + "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "indigo", "juliet", "kilo", "lima", + "mike", "november", "oscar", "papa", "quebec", "romeo", "sierra", "tango", "uniform", "victor", "whisky", + "xray", "yankee", "zulu" }; + + public static final String[] CREATE_LIST = { "aardvark", "banana", "cucumber", "daffodil", "emu", "frog", "gibbon", + "humour", "injection", "jelly", "key", "lemur", "monkey", "number", "open", "plummet", "quest", + "replication", "steam", "tunnel", "uncommon", "verbose", "where", "xylem", "yellow", "zebra", }; + + public static final String[] UPDATE_LIST = { "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", + "hotel", "indigo", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa", "quebec", "romeo", + "sierra", "tango", "uniform", "victor", "whisky", "xray", "yankee", "zulu" }; + + public static final String[] CREATE_LIST_2 = { "aardvark2", "banana2", "cucumber2", "daffodil2", "emu2", "frog2", "gibbon2", + "humour2", "injection2", "jelly2", "key2", "lemur2", "monkey2", "number2", "open2", "plummet2", "quest2", + "replication2", "steam2", "tunnel2", "uncommon2", "verbose2", "where2", "xylem2", "yellow2", "zebra2", }; + +public static final String[] UPDATE_LIST_2 = { "alpha2", "bravo2", "charlie2", "delta2", "echo2", "foxtrot2", "golf2", + "hotel2", "indigo2", "juliet2", "kilo2", "lima2", "mike2", "november2", "oscar2", "papa2", "quebec2", "romeo2", + "sierra2", "tango2", "uniform2", "victor2", "whisky2", "xray2", "yankee2", "zulu2" }; + + public IndexInfoTest() + { + super(); + } + + public IndexInfoTest(String arg0) + { + super(arg0); + } + + public void testCreateAndSearch() throws IOException + { + System.setProperty("disableLuceneLocks", "true"); + + // no deletions - create only + HashSet deletions = new HashSet(); + for (int i = 0; i < 0; i++) + { + deletions.add(new NodeRef(new StoreRef("woof", "bingle"), GUID.generate())); + } + + File tempLocation = TempFileProvider.getTempDir(); + File testArea = new File(tempLocation, "IndexInfoTest"); + File testDir = new File(testArea, "" + System.currentTimeMillis()); + final IndexInfo ii = new IndexInfo(testDir); + + for (int i = 0; i < WORD_LIST.length; i++) + { + IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), i); + reader.close(); + + String guid = GUID.generate(); + ii.setStatus(guid, TransactionStatus.ACTIVE, null, null); + IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer()); + + Document doc = new Document(); + for (int k = 0; k < 15; k++) + { + doc.add(new Field("ID" + k, guid, false, true, false)); + } + doc.add(new Field("TEXT", WORD_LIST[i], false, true, false)); + writer.addDocument(doc); + + ii.closeDeltaIndexWriter(guid); + ii.setStatus(guid, TransactionStatus.PREPARING, null, null); + ii.setPreparedState(guid, deletions, 1); + ii.getDeletions(guid); + ii.setStatus(guid, TransactionStatus.PREPARED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), i); + for (int j = 0; j < WORD_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", WORD_LIST[j])); + if (j < i) + { + assertTrue(tds.next()); + assertEquals(tds.doc(), j); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions); + assertEquals(reader.numDocs(), i + 1); + for (int j = 0; j < WORD_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", WORD_LIST[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertEquals(tds.doc(), j); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + ii.setStatus(guid, TransactionStatus.COMMITTING, null, null); + ii.setStatus(guid, TransactionStatus.COMMITTED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), i + 1); + for (int j = 0; j < WORD_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", WORD_LIST[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertEquals(tds.doc(), j); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + } + + } + + public void testCreateDeleteAndSearch() throws IOException + { + assertEquals(CREATE_LIST.length, UPDATE_LIST.length); + + StoreRef storeRef = new StoreRef("woof", "bingle"); + + System.setProperty("disableLuceneLocks", "true"); + + // no deletions - create only + ArrayList nodeRefs = new ArrayList(); + + File tempLocation = TempFileProvider.getTempDir(); + File testArea = new File(tempLocation, "IndexInfoTest"); + File testDir = new File(testArea, "" + System.currentTimeMillis()); + final IndexInfo ii = new IndexInfo(testDir); + + for (int i = 0; i < CREATE_LIST.length; i++) + { + IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), i); + reader.close(); + + String guid = GUID.generate(); + ii.setStatus(guid, TransactionStatus.ACTIVE, null, null); + IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer()); + + Document doc = new Document(); + for (int k = 0; k < 15; k++) + { + doc.add(new Field("ID" + k, guid, false, true, false)); + } + doc.add(new Field("TEXT", CREATE_LIST[i], false, true, false)); + NodeRef nodeRef = new NodeRef(storeRef, GUID.generate()); + nodeRefs.add(nodeRef); + doc.add(new Field("ID", nodeRef.toString(), false, true, false)); + writer.addDocument(doc); + + ii.closeDeltaIndexWriter(guid); + ii.setStatus(guid, TransactionStatus.PREPARING, null, null); + ii.setPreparedState(guid, new HashSet(), 1); + ii.getDeletions(guid); + ii.setStatus(guid, TransactionStatus.PREPARED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), i); + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j < i) + { + assertTrue(tds.next()); + assertEquals(tds.doc(), j); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet()); + assertEquals(reader.numDocs(), i + 1); + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertEquals(tds.doc(), j); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + ii.setStatus(guid, TransactionStatus.COMMITTING, null, null); + ii.setStatus(guid, TransactionStatus.COMMITTED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), i + 1); + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertEquals(tds.doc(), j); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + } + + for (int i = 0; i < CREATE_LIST.length; i++) + { + HashSet deletions = new HashSet(); + deletions.add(nodeRefs.get(i)); + + IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), CREATE_LIST.length - i); + reader.close(); + + String guid = GUID.generate(); + ii.setStatus(guid, TransactionStatus.ACTIVE, null, null); + ii.closeDeltaIndexWriter(guid); + ii.setStatus(guid, TransactionStatus.PREPARING, null, null); + ii.setPreparedState(guid, deletions, 1); + ii.getDeletions(guid); + ii.setStatus(guid, TransactionStatus.PREPARED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), CREATE_LIST.length - i); + int lastDoc = -1; + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j >= i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions); + assertEquals(reader.numDocs(), UPDATE_LIST.length - i - 1); + lastDoc = -1; + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j > i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + + reader.close(); + + ii.setStatus(guid, TransactionStatus.COMMITTING, null, null); + ii.setStatus(guid, TransactionStatus.COMMITTED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), UPDATE_LIST.length - i - 1); + lastDoc = -1; + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j > i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + + reader.close(); + + IndexReader reader1 = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + IndexReader reader2 = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + IndexReader reader3 = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + reader3.close(); + reader2.close(); + reader1.close(); + + } + + } + + public void testCreateUpdateAndSearch() throws IOException + { + assertEquals(CREATE_LIST.length, UPDATE_LIST.length); + + StoreRef storeRef = new StoreRef("woof", "bingle"); + + System.setProperty("disableLuceneLocks", "true"); + + // no deletions - create only + ArrayList nodeRefs = new ArrayList(); + + File tempLocation = TempFileProvider.getTempDir(); + File testArea = new File(tempLocation, "IndexInfoTest"); + File testDir = new File(testArea, "" + System.currentTimeMillis()); + final IndexInfo ii = new IndexInfo(testDir); + + for (int i = 0; i < CREATE_LIST.length; i++) + { + IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), i); + reader.close(); + + String guid = GUID.generate(); + ii.setStatus(guid, TransactionStatus.ACTIVE, null, null); + IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer()); + + Document doc = new Document(); + for (int k = 0; k < 15; k++) + { + doc.add(new Field("ID" + k, guid, false, true, false)); + } + doc.add(new Field("TEXT", CREATE_LIST[i], false, true, false)); + NodeRef nodeRef = new NodeRef(storeRef, GUID.generate()); + nodeRefs.add(nodeRef); + doc.add(new Field("ID", nodeRef.toString(), false, true, false)); + writer.addDocument(doc); + + ii.closeDeltaIndexWriter(guid); + ii.setStatus(guid, TransactionStatus.PREPARING, null, null); + ii.setPreparedState(guid, new HashSet(), 1); + ii.getDeletions(guid); + ii.setStatus(guid, TransactionStatus.PREPARED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), i); + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j < i) + { + assertTrue(tds.next()); + assertEquals(tds.doc(), j); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet()); + assertEquals(reader.numDocs(), i + 1); + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertEquals(tds.doc(), j); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + ii.setStatus(guid, TransactionStatus.COMMITTING, null, null); + ii.setStatus(guid, TransactionStatus.COMMITTED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), i + 1); + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertEquals(tds.doc(), j); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + } + + for (int i = 0; i < UPDATE_LIST.length; i++) + { + HashSet deletions = new HashSet(); + deletions.add(nodeRefs.get(i)); + + IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), UPDATE_LIST.length); + reader.close(); + + String guid = GUID.generate(); + ii.setStatus(guid, TransactionStatus.ACTIVE, null, null); + IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer()); + + Document doc = new Document(); + for (int k = 0; k < 15; k++) + { + doc.add(new Field("ID" + k, guid, false, true, false)); + } + doc.add(new Field("TEXT", UPDATE_LIST[i], false, true, false)); + writer.addDocument(doc); + + ii.closeDeltaIndexWriter(guid); + ii.setStatus(guid, TransactionStatus.PREPARING, null, null); + ii.setPreparedState(guid, deletions, 1); + ii.getDeletions(guid); + ii.setStatus(guid, TransactionStatus.PREPARED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), UPDATE_LIST.length); + int lastDoc = -1; + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j >= i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + for (int j = 0; j < UPDATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", UPDATE_LIST[j])); + if (j < i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions); + assertEquals(reader.numDocs(), UPDATE_LIST.length); + lastDoc = -1; + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j > i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + for (int j = 0; j < UPDATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", UPDATE_LIST[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + + reader.close(); + + ii.setStatus(guid, TransactionStatus.COMMITTING, null, null); + ii.setStatus(guid, TransactionStatus.COMMITTED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + assertEquals(reader.numDocs(), UPDATE_LIST.length); + lastDoc = -1; + for (int j = 0; j < CREATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", CREATE_LIST[j])); + if (j > i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + for (int j = 0; j < UPDATE_LIST.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", UPDATE_LIST[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + } + + } + + public void testMultiThreadedCreateAndSearch() + { + + System.setProperty("disableLuceneLocks", "true"); + + File tempLocation = TempFileProvider.getTempDir(); + File testArea = new File(tempLocation, "IndexInfoTest"); + File testDir = new File(testArea, "" + System.currentTimeMillis()); + final IndexInfo ii = new IndexInfo(testDir); + + Thread thread1 = new Thread(new Test(ii, CREATE_LIST, UPDATE_LIST)); + Thread thread2 = new Thread(new Test(ii, CREATE_LIST_2, UPDATE_LIST_2)); + thread1.start(); + thread2.start(); + try + { + thread1.join(); + thread2.join(); + } + catch (InterruptedException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + public static class Test implements Runnable + { + String[] create; + String[] update; + IndexInfo ii; + + Test(IndexInfo ii, String[] create, String[] update) + { + this.ii = ii; + this.create = create; + this.update = update; + } + + public void run() + { + try + { + assertEquals(create.length, update.length); + + StoreRef storeRef = new StoreRef("woof", "bingle"); + + // no deletions - create only + ArrayList nodeRefs = new ArrayList(); + + for (int i = 0; i < create.length; i++) + { + IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + reader.close(); + + String guid = GUID.generate(); + ii.setStatus(guid, TransactionStatus.ACTIVE, null, null); + IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer()); + + Document doc = new Document(); + for (int k = 0; k < 15; k++) + { + doc.add(new Field("ID" + k, guid, false, true, false)); + } + doc.add(new Field("TEXT", create[i], false, true, false)); + NodeRef nodeRef = new NodeRef(storeRef, GUID.generate()); + nodeRefs.add(nodeRef); + doc.add(new Field("ID", nodeRef.toString(), false, true, false)); + writer.addDocument(doc); + + ii.closeDeltaIndexWriter(guid); + ii.setStatus(guid, TransactionStatus.PREPARING, null, null); + ii.setPreparedState(guid, new HashSet(), 1); + ii.getDeletions(guid); + ii.setStatus(guid, TransactionStatus.PREPARED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + + int lastDoc = -1; + + for (int j = 0; j < create.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", create[j])); + if (j < i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, new HashSet()); + lastDoc = -1; + for (int j = 0; j < create.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", create[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + ii.setStatus(guid, TransactionStatus.COMMITTING, null, null); + ii.setStatus(guid, TransactionStatus.COMMITTED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + lastDoc = -1; + for (int j = 0; j < create.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", create[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + } + + for (int i = 0; i < update.length; i++) + { + HashSet deletions = new HashSet(); + deletions.add(nodeRefs.get(i)); + + IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + + reader.close(); + + String guid = GUID.generate(); + ii.setStatus(guid, TransactionStatus.ACTIVE, null, null); + IndexWriter writer = ii.getDeltaIndexWriter(guid, new StandardAnalyzer()); + + Document doc = new Document(); + for (int k = 0; k < 15; k++) + { + doc.add(new Field("ID" + k, guid, false, true, false)); + } + doc.add(new Field("TEXT", update[i], false, true, false)); + writer.addDocument(doc); + + ii.closeDeltaIndexWriter(guid); + ii.setStatus(guid, TransactionStatus.PREPARING, null, null); + ii.setPreparedState(guid, deletions, 1); + ii.getDeletions(guid); + ii.setStatus(guid, TransactionStatus.PREPARED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + + int lastDoc = -1; + for (int j = 0; j < create.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", create[j])); + if (j >= i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + for (int j = 0; j < update.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", update[j])); + if (j < i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(guid, deletions); + + lastDoc = -1; + for (int j = 0; j < create.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", create[j])); + if (j > i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + for (int j = 0; j < update.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", update[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + + reader.close(); + + ii.setStatus(guid, TransactionStatus.COMMITTING, null, null); + ii.setStatus(guid, TransactionStatus.COMMITTED, null, null); + + reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); + + lastDoc = -1; + for (int j = 0; j < create.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", create[j])); + if (j > i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + for (int j = 0; j < update.length; j++) + { + TermDocs tds = reader.termDocs(new Term("TEXT", update[j])); + if (j <= i) + { + assertTrue(tds.next()); + assertTrue(tds.doc() > lastDoc); + lastDoc = tds.doc(); + } + else + { + assertFalse(tds.next()); + } + } + reader.close(); + + } + + } + catch (IOException e) + { + System.exit(-1); + } + } + + } +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexType.java b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexType.java index e690641b13..cc2bb9df74 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexType.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexType.java @@ -12,30 +12,19 @@ public enum IndexType */ INDEX, - /** - * Identifies a subindex. This will also be optimised. Sub-indexes are periodically merged into the index. - */ - SUBINDEX, - /** * An overlay. This is an optimised index with a deletion list. To commit an overlay requires no deletions against other indexes. Deletions are done when an overlay turns - * into or is merged into a subindex. Overlays are periodically merged into a sub index. An overlay can require or have background properties indexed - */ - INDEX_OVERLAY, - - /** - * A long running overlay defintion against the index. Not yet supported. - * This, itself, may have transactional additions. - */ - OVERLAY, - - /** - * A delta is a transactional change set. This commits to an overlay index. + * into or is merged into a index. Overlays are periodically merged into an index. An overlay can require or have background properties indexed. */ DELTA, /** - * A delta to an overlay + * A long running overlay definition against the index. Not yet supported. + * This, itself, may have transactional additions. */ + OVERLAY, + OVERLAY_DELTA; + + } \ No newline at end of file diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCounting.java b/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCounting.java index 81fe4cf3fe..a809e14aee 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCounting.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCounting.java @@ -24,7 +24,7 @@ public interface ReferenceCounting public void decrementReferenceCount() throws IOException; - public boolean isUsed(); + public int getReferenceCount(); public void setInvalidForReuse() throws IOException; } \ No newline at end of file diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java b/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java index ca2f544970..5c95d2645c 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java @@ -18,70 +18,52 @@ package org.alfresco.repo.search.impl.lucene.index; import java.io.IOException; -import org.aopalliance.intercept.MethodInvocation; +import org.apache.log4j.Logger; +import org.apache.lucene.index.FilterIndexReader; import org.apache.lucene.index.IndexReader; -import org.springframework.aop.framework.ProxyFactory; -import org.springframework.aop.framework.adapter.AdvisorAdapterRegistry; -import org.springframework.aop.framework.adapter.GlobalAdvisorAdapterRegistry; -import org.springframework.aop.support.DelegatingIntroductionInterceptor; -import org.springframework.aop.target.SingletonTargetSource; public class ReferenceCountingReadOnlyIndexReaderFactory { - public static IndexReader createReader(IndexReader indexReader) + public static IndexReader createReader(String id, IndexReader indexReader) { - AdvisorAdapterRegistry advisorAdapterRegistry = GlobalAdvisorAdapterRegistry.getInstance(); - ProxyFactory proxyFactory = new ProxyFactory(); - proxyFactory.addAdvisor(advisorAdapterRegistry.wrap(new Interceptor(indexReader))); - proxyFactory.setTargetSource(new SingletonTargetSource(indexReader)); - IndexReader proxy = (IndexReader) proxyFactory.getProxy(); - return proxy; + return new ReferenceCountingReadOnlyIndexReader(id, indexReader); } - public static class Interceptor extends DelegatingIntroductionInterceptor implements ReferenceCounting + public static class ReferenceCountingReadOnlyIndexReader extends FilterIndexReader implements ReferenceCounting { + private static Logger s_logger = Logger.getLogger(ReferenceCountingReadOnlyIndexReader.class); + private static final long serialVersionUID = 7693185658022810428L; - IndexReader indexReader; - + String id; + int refCount = 0; boolean invalidForReuse = false; - Interceptor(IndexReader indexReader) + ReferenceCountingReadOnlyIndexReader(String id, IndexReader indexReader) { - this.indexReader = indexReader; - } - - public Object invoke(MethodInvocation mi) throws Throwable - { - // Read only - String methodName = mi.getMethod().getName(); - if (methodName.equals("delete") || methodName.equals("doDelete")) - { - throw new UnsupportedOperationException("Delete is not supported by read only index readers"); - } - // Close - else if (methodName.equals("close")) - { - decrementReferenceCount(); - return null; - } - else - { - return super.invoke(mi); - } + super(indexReader); + this.id = id; } public synchronized void incrementReferenceCount() { refCount++; + if(s_logger.isDebugEnabled()) + { + s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " - increment - ref count is "+refCount); + } } public synchronized void decrementReferenceCount() throws IOException { refCount--; + if(s_logger.isDebugEnabled()) + { + s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " - decrement - ref count is "+refCount); + } closeIfRequired(); } @@ -89,19 +71,52 @@ public class ReferenceCountingReadOnlyIndexReaderFactory { if ((refCount == 0) && invalidForReuse) { - indexReader.close(); + if(s_logger.isDebugEnabled()) + { + s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " closed."); + } + in.close(); + } + else + { + if(s_logger.isDebugEnabled()) + { + s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " still open .... ref = "+refCount+" invalidForReuse = "+invalidForReuse); + } } } - public synchronized boolean isUsed() + public synchronized int getReferenceCount() { - return (refCount > 0); + return refCount; } public synchronized void setInvalidForReuse() throws IOException { invalidForReuse = true; + if(s_logger.isDebugEnabled()) + { + s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " set invalid for reuse"); + } closeIfRequired(); } + + @Override + protected void doClose() throws IOException + { + if(s_logger.isDebugEnabled()) + { + s_logger.debug(Thread.currentThread().getName()+ ": Reader "+id+ " closing"); + } + decrementReferenceCount(); + } + + @Override + protected void doDelete(int n) throws IOException + { + throw new UnsupportedOperationException("Delete is not supported by read only index readers"); + } + + } } diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/TransactionStatus.java b/source/java/org/alfresco/repo/search/impl/lucene/index/TransactionStatus.java index 6390e61e37..07413aebf8 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/TransactionStatus.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/TransactionStatus.java @@ -21,17 +21,48 @@ package org.alfresco.repo.search.impl.lucene.index; public enum TransactionStatus { + // Match the order in javax.transaction.Status so ordinal values are correct ACTIVE { - public boolean follows(TransactionStatus previous) + public boolean isCommitted() { return false; } + + public boolean isTransient() + { + return true; + } + + public boolean canBeReordered() + { + return true; + } + + public boolean follows(TransactionStatus previous) + { + return previous == null; + } }, MARKED_ROLLBACK { + public boolean isCommitted() + { + return false; + } + + public boolean isTransient() + { + return true; + } + + public boolean canBeReordered() + { + return true; + } + public boolean follows(TransactionStatus previous) { return previous.allowsRollbackOrMark(previous); @@ -40,6 +71,21 @@ public enum TransactionStatus PREPARED { + public boolean isCommitted() + { + return false; + } + + public boolean isTransient() + { + return false; + } + + public boolean canBeReordered() + { + return false; + } + public boolean follows(TransactionStatus previous) { return previous == TransactionStatus.PREPARING; @@ -53,6 +99,16 @@ public enum TransactionStatus return true; } + public boolean isTransient() + { + return false; + } + + public boolean canBeReordered() + { + return false; + } + public boolean follows(TransactionStatus previous) { return previous == TransactionStatus.COMMITTING; @@ -61,6 +117,21 @@ public enum TransactionStatus ROLLEDBACK { + public boolean isCommitted() + { + return false; + } + + public boolean isTransient() + { + return true; + } + + public boolean canBeReordered() + { + return true; + } + public boolean follows(TransactionStatus previous) { return previous == TransactionStatus.ROLLINGBACK; @@ -69,6 +140,21 @@ public enum TransactionStatus UNKNOWN { + public boolean isCommitted() + { + return false; + } + + public boolean isTransient() + { + return true; + } + + public boolean canBeReordered() + { + return true; + } + public boolean follows(TransactionStatus previous) { return false; @@ -77,6 +163,21 @@ public enum TransactionStatus NO_TRANSACTION { + public boolean isCommitted() + { + return false; + } + + public boolean isTransient() + { + return true; + } + + public boolean canBeReordered() + { + return true; + } + public boolean follows(TransactionStatus previous) { return false; @@ -85,6 +186,21 @@ public enum TransactionStatus PREPARING { + public boolean isCommitted() + { + return false; + } + + public boolean isTransient() + { + return true; + } + + public boolean canBeReordered() + { + return true; + } + public boolean follows(TransactionStatus previous) { return previous == TransactionStatus.ACTIVE; @@ -93,6 +209,21 @@ public enum TransactionStatus COMMITTING { + public boolean isCommitted() + { + return false; + } + + public boolean isTransient() + { + return true; + } + + public boolean canBeReordered() + { + return false; + } + public boolean follows(TransactionStatus previous) { return previous == TransactionStatus.PREPARED; @@ -101,6 +232,21 @@ public enum TransactionStatus ROLLINGBACK { + public boolean isCommitted() + { + return false; + } + + public boolean isTransient() + { + return true; + } + + public boolean canBeReordered() + { + return true; + } + public boolean follows(TransactionStatus previous) { return previous.allowsRollbackOrMark(previous); @@ -117,6 +263,16 @@ public enum TransactionStatus return true; } + public boolean isTransient() + { + return false; + } + + public boolean canBeReordered() + { + return false; + } + public boolean follows(TransactionStatus previous) { return false; @@ -128,6 +284,21 @@ public enum TransactionStatus */ MERGE_TARGET { + public boolean isCommitted() + { + return false; + } + + public boolean isTransient() + { + return false; + } + + public boolean canBeReordered() + { + return false; + } + public boolean follows(TransactionStatus previous) { return false; @@ -137,67 +308,108 @@ public enum TransactionStatus /* * These index overlays require reindexing */ - COMMITTED_REQUIRES_REINDEX - { - public boolean isCommitted() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return false; - } - }, +// COMMITTED_REQUIRES_REINDEX +// { +// public boolean isCommitted() +// { +// return true; +// } +// +// public boolean isTransient() +// { +// return false; +// } +// +// public boolean canBeReordered() +// { +// return false; +// } +// +// public boolean follows(TransactionStatus previous) +// { +// return false; +// } +// }, /* * These index overlays are reindexing */ - COMMITTED_REINDEXING - { - public boolean isCommitted() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return false; - } - }, +// COMMITTED_REINDEXING +// { +// public boolean isCommitted() +// { +// return true; +// } +// +// +// public boolean canBeReordered() +// { +// return false; +// } +// +// public boolean isTransient() +// { +// return false; +// } +// +// public boolean follows(TransactionStatus previous) +// { +// return false; +// } +// }, /* * These index overlays have ben reindexed. */ - COMMITTED_REINDEXED - { - public boolean isCommitted() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return false; - } - }, +// COMMITTED_REINDEXED +// { +// public boolean isCommitted() +// { +// return true; +// } +// +// public boolean isTransient() +// { +// return false; +// } +// +// public boolean canBeReordered() +// { +// return false; +// } +// +// public boolean follows(TransactionStatus previous) +// { +// return false; +// } +// }, /* * Committed but the index still has deletions */ - COMMITTED_WITH_DELETIONS - { - public boolean isCommitted() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return false; - } - }, +// COMMITTED_WITH_DELETIONS +// { +// public boolean isCommitted() +// { +// return true; +// } +// +// public boolean isTransient() +// { +// return false; +// } +// +// public boolean canBeReordered() +// { +// return false; +// } +// +// public boolean follows(TransactionStatus previous) +// { +// return false; +// } +// }, /* * Pending deleted are being committed to for the delta. @@ -209,6 +421,16 @@ public enum TransactionStatus return true; } + public boolean isTransient() + { + return false; + } + + public boolean canBeReordered() + { + return false; + } + public boolean follows(TransactionStatus previous) { return false; @@ -220,16 +442,32 @@ public enum TransactionStatus */ DELETABLE { - public boolean follows(TransactionStatus previous) + public boolean isCommitted() { return false; } + + public boolean isTransient() + { + return false; + } + + public boolean canBeReordered() + { + return false; + } + + public boolean follows(TransactionStatus previous) + { + return true; + } }; - public boolean isCommitted() - { - return false; - } + public abstract boolean isCommitted(); + + public abstract boolean isTransient(); + + public abstract boolean canBeReordered(); public abstract boolean follows(TransactionStatus previous);