From ad23a594f55de260038d870e6598fe5a96166247 Mon Sep 17 00:00:00 2001 From: Derek Hulley Date: Mon, 6 Oct 2008 23:06:54 +0000 Subject: [PATCH] Merged V2.2 to HEAD 11016: Query performance improvements 11018: Build fixes after .... Query performance improvements 11043: Updated Lucene config and test fixes 11047: Fixed test's expected results 11049: Build fix 11051: Tighten up on Auditable tests and checks for null git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@11221 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261 --- .../repo/audit/AuditableAspectTest.java | 17 +- .../repo/domain/AuditableProperties.java | 5 +- .../HibernateNodeDaoServiceImpl.java | 19 +- .../repo/search/SearcherComponentTest.java | 10 +- .../search/impl/lucene/ADMLuceneTest.java | 91 +- ...stractLuceneIndexerAndSearcherFactory.java | 233 +++- .../repo/search/impl/lucene/LuceneConfig.java | 114 ++ .../impl/lucene/index/CachingIndexReader.java | 2 + .../search/impl/lucene/index/IndexInfo.java | 205 +--- ...nceCountingReadOnlyIndexReaderFactory.java | 1088 ++++------------- .../search/impl/lucene/query/LeafScorer.java | 69 +- 11 files changed, 777 insertions(+), 1076 deletions(-) diff --git a/source/java/org/alfresco/repo/audit/AuditableAspectTest.java b/source/java/org/alfresco/repo/audit/AuditableAspectTest.java index eb75bd1e99..e7158f7177 100644 --- a/source/java/org/alfresco/repo/audit/AuditableAspectTest.java +++ b/source/java/org/alfresco/repo/audit/AuditableAspectTest.java @@ -104,13 +104,26 @@ public class AuditableAspectTest extends BaseSpringTest QName.createQName("{test}testperson"), ContentModel.TYPE_PERSON, personProps); + NodeRef nodeRef = childAssocRef.getChildRef(); // Assert the person is not auditable - Set aspects = nodeService.getAspects(childAssocRef.getChildRef()); - assertFalse(aspects.contains(ContentModel.ASPECT_AUDITABLE)); + Set aspects = nodeService.getAspects(nodeRef); + assertFalse("cm:auditable must not be present.", aspects.contains(ContentModel.ASPECT_AUDITABLE)); + Map properties = nodeService.getProperties(nodeRef); + assertFalse("cm:creator must not be present", properties.containsKey(ContentModel.PROP_CREATOR)); + assertFalse("cm:created must not be present", properties.containsKey(ContentModel.PROP_CREATED)); + + assertNull( + "Didn't expect to get single auditable property", + nodeService.getProperty(nodeRef, ContentModel.PROP_CREATOR)); System.out.println(NodeStoreInspector.dumpNodeStore(nodeService, storeRef)); } + + public void test() throws Throwable + { + + } public void testAddAudit() diff --git a/source/java/org/alfresco/repo/domain/AuditableProperties.java b/source/java/org/alfresco/repo/domain/AuditableProperties.java index fe294af887..c7bb5a2e59 100644 --- a/source/java/org/alfresco/repo/domain/AuditableProperties.java +++ b/source/java/org/alfresco/repo/domain/AuditableProperties.java @@ -92,11 +92,12 @@ public class AuditableProperties } else if (qname.equals(ContentModel.PROP_MODIFIER)) { - return auditModifier; + return auditModifier == null ? auditCreator : auditModifier; } else if (qname.equals(ContentModel.PROP_MODIFIED)) { - return DefaultTypeConverter.INSTANCE.convert(Date.class, auditModified); + String dateStr = auditModified == null ? auditCreated : auditModified; + return DefaultTypeConverter.INSTANCE.convert(Date.class, dateStr); } else if (qname.equals(ContentModel.PROP_ACCESSED)) { diff --git a/source/java/org/alfresco/repo/node/db/hibernate/HibernateNodeDaoServiceImpl.java b/source/java/org/alfresco/repo/node/db/hibernate/HibernateNodeDaoServiceImpl.java index 004edabba5..91ca5b1f2d 100644 --- a/source/java/org/alfresco/repo/node/db/hibernate/HibernateNodeDaoServiceImpl.java +++ b/source/java/org/alfresco/repo/node/db/hibernate/HibernateNodeDaoServiceImpl.java @@ -1020,8 +1020,16 @@ public class HibernateNodeDaoServiceImpl extends HibernateDaoSupport implements // Handle cm:auditable if (AuditableProperties.isAuditableProperty(propertyQName)) { - AuditableProperties auditableProperties = node.getAuditableProperties(); - return auditableProperties.getAuditableProperty(propertyQName); + // Only bother if the aspect is present + if (hasNodeAspect(node, ContentModel.ASPECT_AUDITABLE)) + { + AuditableProperties auditableProperties = node.getAuditableProperties(); + return auditableProperties.getAuditableProperty(propertyQName); + } + else + { + return null; + } } QNameEntity propertyQNameEntity = qnameDAO.getQNameEntity(propertyQName); @@ -1051,8 +1059,11 @@ public class HibernateNodeDaoServiceImpl extends HibernateDaoSupport implements dictionaryService); // Handle cm:auditable - AuditableProperties auditableProperties = node.getAuditableProperties(); - converted.putAll(auditableProperties.getAuditableProperties()); + if (hasNodeAspect(node, ContentModel.ASPECT_AUDITABLE)) + { + AuditableProperties auditableProperties = node.getAuditableProperties(); + converted.putAll(auditableProperties.getAuditableProperties()); + } // Done return converted; diff --git a/source/java/org/alfresco/repo/search/SearcherComponentTest.java b/source/java/org/alfresco/repo/search/SearcherComponentTest.java index 9541eefd97..87fc2404c6 100644 --- a/source/java/org/alfresco/repo/search/SearcherComponentTest.java +++ b/source/java/org/alfresco/repo/search/SearcherComponentTest.java @@ -153,7 +153,7 @@ public class SearcherComponentTest extends TestCase xpath = new NodeServiceXPath("*/*", documentNavigator, null); list = xpath.selectNodes(new ChildAssociationRef(null, null, null, rootNodeRef)); - assertEquals(4, list.size()); + assertEquals(5, list.size()); xpath = new NodeServiceXPath("*/*/*", documentNavigator, null); list = xpath.selectNodes(new ChildAssociationRef(null, null, null, rootNodeRef)); @@ -169,7 +169,7 @@ public class SearcherComponentTest extends TestCase xpath = new NodeServiceXPath("*//.", documentNavigator, null); list = xpath.selectNodes(new ChildAssociationRef(null, null, null, rootNodeRef)); - assertEquals(12, list.size()); + assertEquals(13, list.size()); xpathStr = "test:root_p_n1"; xpath = new NodeServiceXPath(xpathStr, documentNavigator, null); @@ -268,7 +268,7 @@ public class SearcherComponentTest extends TestCase xpath = new NodeServiceXPath(xpathStr, documentNavigator, new QueryParameterDefinition[] { paramDef }); xpath.addNamespace(BaseNodeServiceTest.TEST_PREFIX, BaseNodeServiceTest.NAMESPACE); list = xpath.selectNodes(assocRefs.get(qname)); - assertEquals(2, list.size()); // 2 distinct paths to node n8, which is of type content + assertEquals(3, list.size()); // 2 distinct paths to node n8, which is of type content xpath = new NodeServiceXPath("/", documentNavigator, null); xpath.addNamespace(BaseNodeServiceTest.TEST_PREFIX, BaseNodeServiceTest.NAMESPACE); @@ -560,7 +560,7 @@ public class SearcherComponentTest extends TestCase "element\\(\\s*(\\*|\\$?\\w*:\\w*)\\s*,\\s*(\\*|\\$?\\w*:\\w*)\\s*\\)", "$1[subtypeOf(\"$2\")]"), documentNavigator, null); list = xpath.selectNodes(new ChildAssociationRef(null, null, null, rootNodeRef)); - assertEquals(12, list.size()); + assertEquals(13, list.size()); xpath = new NodeServiceXPath("//element(jcr:root, *)".replaceAll( "element\\(\\s*(\\*|\\$?\\w*:\\w*)\\s*,\\s*(\\*|\\$?\\w*:\\w*)\\s*\\)", "$1[subtypeOf(\"$2\")]"), @@ -577,7 +577,7 @@ public class SearcherComponentTest extends TestCase "element\\(\\s*(\\*|\\$?\\w*:\\w*)\\s*,\\s*(\\*|\\$?\\w*:\\w*)\\s*\\)", "$1[subtypeOf(\"$2\")]"), documentNavigator, new QueryParameterDefinition[] { paramDef }); list = xpath.selectNodes(new ChildAssociationRef(null, null, null, rootNodeRef)); - assertEquals(2, list.size()); + assertEquals(3, list.size()); paramDef = new QueryParameterDefImpl(QName.createQName("test:type", namespacePrefixResolver), dictionaryService .getDataType(DataTypeDefinition.QNAME), true, BaseNodeServiceTest.TYPE_QNAME_TEST_CONTENT diff --git a/source/java/org/alfresco/repo/search/impl/lucene/ADMLuceneTest.java b/source/java/org/alfresco/repo/search/impl/lucene/ADMLuceneTest.java index 5a56ec0ad0..49956a537a 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/ADMLuceneTest.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/ADMLuceneTest.java @@ -33,7 +33,9 @@ import java.io.ObjectOutputStream; import java.io.Serializable; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; import java.util.Date; +import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; import java.util.Locale; @@ -109,7 +111,7 @@ public class ADMLuceneTest extends TestCase { private static final String TEST_NAMESPACE = "http://www.alfresco.org/test/lucenetest"; - + private static final QName ASSOC_TYPE_QNAME = QName.createQName(TEST_NAMESPACE, "assoc"); private static ApplicationContext ctx = ApplicationContextHelper.getApplicationContext(); @@ -420,10 +422,12 @@ public class ADMLuceneTest extends TestCase documentOrder = new NodeRef[] { rootNodeRef, n4, n5, n6, n7, n8, n9, n10, n11, n12, n13, n14, n3, n1, n2 }; - nodeService.addAspect(n3, ContentModel.ASPECT_AUDITABLE, null); - nodeService.addAspect(n1, ContentModel.ASPECT_AUDITABLE, null); +// TODO: Why was the cm:auditable aspect added here? +// By adding it, the auditable properties were set automatically. +// nodeService.addAspect(n3, ContentModel.ASPECT_AUDITABLE, null); +// nodeService.addAspect(n1, ContentModel.ASPECT_AUDITABLE, null); nodeService.setProperty(n1, ContentModel.PROP_MODIFIED, new Date(new Date().getTime() - 1000*60*60)); - nodeService.addAspect(n2, ContentModel.ASPECT_AUDITABLE, null); +// nodeService.addAspect(n2, ContentModel.ASPECT_AUDITABLE, null); } private double orderDoubleCount = -0.11d; @@ -477,13 +481,17 @@ public class ADMLuceneTest extends TestCase public void restManyReaders() throws Exception { + QName propQName = QName.createQName(TEST_NAMESPACE, "text-indexed-stored-tokenised-atomic"); + NodeRef base = rootNodeRef; for (int i = 0; i < 10; i++) { NodeRef dir = nodeService.createNode(base, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}d-" + i), testSuperType, null).getChildRef(); - for (int j = 0; j < 10; j++) + for (int j = 0; j < 100; j++) { - NodeRef file = nodeService.createNode(dir, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}meep"), testSuperType, null).getChildRef(); + Map properties = new HashMap(); + properties.put(propQName, "lemon"); + NodeRef file = nodeService.createNode(dir, ContentModel.ASSOC_CHILDREN, QName.createQName("{namespace}meep"), testSuperType, properties).getChildRef(); } } testTX.commit(); @@ -504,10 +512,23 @@ public class ADMLuceneTest extends TestCase Thread runner = null; - for (int i = 0; i < 20; i++) + // testQuery(searcher, runner, "PATH:\"/d-0/*\""); + // testQuery(searcher, runner, "PATH:\"/d-0/meep\""); + // testQuery(searcher, runner, "PATH:\"/d-0//*\""); + // testQuery(searcher, runner, "PATH:\"/d-0//meep\""); + testQuery(searcher, runner, "PATH:\"//*\""); + // testQuery(searcher, runner, "PATH:\"//meep\""); + // testQuery(searcher, runner, "@"+LuceneQueryParser.escape(propQName.toString())+":\"lemon\""); + + } + + private void testQuery(ADMLuceneSearcherImpl searcher, Thread runner, String query) + { + for (int i = 0; i < 1; i++) { - runner = new QueryThread("Concurrent-" + i, runner, searcher); + runner = new QueryThread("Concurrent-" + i, runner, searcher, query); } + long start = System.nanoTime(); if (runner != null) { runner.start(); @@ -521,6 +542,8 @@ public class ADMLuceneTest extends TestCase e.printStackTrace(); } } + long end = System.nanoTime(); + System.out.println(query + "\t" + ((end - start) / 1e9f)); } class QueryThread extends Thread @@ -529,12 +552,15 @@ public class ADMLuceneTest extends TestCase ADMLuceneSearcherImpl searcher; - QueryThread(String name, Thread waiter, ADMLuceneSearcherImpl searcher) + String query; + + QueryThread(String name, Thread waiter, ADMLuceneSearcherImpl searcher, String query) { super(name); this.setDaemon(true); this.waiter = waiter; this.searcher = searcher; + this.query = query; } public void run() @@ -546,7 +572,7 @@ public class ADMLuceneTest extends TestCase } try { - System.out.println("Start " + this.getName()); + // System.out.println("Start " + this.getName()); RetryingTransactionCallback createAndDeleteCallback = new RetryingTransactionCallback() { @@ -555,9 +581,9 @@ public class ADMLuceneTest extends TestCase SessionSizeResourceManager.setDisableInTransaction(); for (int i = 0; i < 100; i++) { - ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "PATH:\"//meep\""); + ResultSet results = searcher.query(rootNodeRef.getStoreRef(), "lucene", query); int count = results.length(); - for(ResultSetRow row : results) + for (ResultSetRow row : results) { NodeRef nr = row.getNodeRef(); } @@ -568,7 +594,7 @@ public class ADMLuceneTest extends TestCase }; retryingTransactionHelper.doInTransaction(createAndDeleteCallback); - System.out.println("End " + this.getName()); + // System.out.println("End " + this.getName()); } catch (Exception e) { @@ -2002,7 +2028,16 @@ public class ADMLuceneTest extends TestCase for (ResultSetRow row : results) { Date currentBun = DefaultTypeConverter.INSTANCE.convert(Date.class, nodeService.getProperty(row.getNodeRef(), ContentModel.PROP_MODIFIED)); - //System.out.println("A " + currentBun + " "+row.getQName()); + if (currentBun != null) + { + Calendar c = new GregorianCalendar(); + c.setTime(currentBun); + c.set(Calendar.MILLISECOND, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.HOUR_OF_DAY, 0); + currentBun = c.getTime(); + } if (date != null) { assertTrue(date.compareTo(currentBun) <= 0); @@ -2023,6 +2058,17 @@ public class ADMLuceneTest extends TestCase { Date currentBun = DefaultTypeConverter.INSTANCE.convert(Date.class, nodeService.getProperty(row.getNodeRef(), ContentModel.PROP_MODIFIED)); // System.out.println(currentBun); + if (currentBun != null) + { + Calendar c = new GregorianCalendar(); + c.setTime(currentBun); + c.set(Calendar.MILLISECOND, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.HOUR_OF_DAY, 0); + currentBun = c.getTime(); + } + if ((date != null) && (currentBun != null)) { assertTrue(date.compareTo(currentBun) >= 0); @@ -3099,20 +3145,17 @@ public class ADMLuceneTest extends TestCase results.close(); // short and long field ranges - + sDate = df.format(date); - results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@cm\\:created:[MIN TO " + sDate + "]", - null, null); - assertEquals(4, results.length()); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@cm\\:created:[MIN TO " + sDate + "]", null, null); + assertEquals(1, results.length()); results.close(); - + sDate = df.format(date); - results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(ContentModel.PROP_CREATED) + ":[MIN TO " + sDate + "]", - null, null); - assertEquals(4, results.length()); + results = searcher.query(rootNodeRef.getStoreRef(), "lucene", "\\@" + escapeQName(ContentModel.PROP_CREATED) + ":[MIN TO " + sDate + "]", null, null); + assertEquals(1, results.length()); results.close(); - - + // Date ranges // Test date collapses but date time does not diff --git a/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java b/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java index 624fad8cac..2986148e11 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java @@ -46,7 +46,6 @@ import javax.transaction.xa.XAResource; import javax.transaction.xa.Xid; import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.repo.avm.AVMNodeService; import org.alfresco.repo.domain.hibernate.BulkLoader; import org.alfresco.repo.search.IndexerException; import org.alfresco.repo.search.MLAnalysisMode; @@ -64,6 +63,7 @@ import org.alfresco.service.transaction.TransactionService; import org.alfresco.util.GUID; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.store.Lock; import org.quartz.Job; @@ -126,7 +126,7 @@ public abstract class AbstractLuceneIndexerAndSearcherFactory implements LuceneI /** the maximum transformation time to allow atomically, defaulting to 20ms */ private long maxAtomicTransformationTime = 20; - private int indexerMaxFieldLength; + private int indexerMaxFieldLength = IndexWriter.DEFAULT_MAX_FIELD_LENGTH; private long writeLockTimeout; @@ -142,6 +142,44 @@ public abstract class AbstractLuceneIndexerAndSearcherFactory implements LuceneI private BulkLoader bulkLoader; + private int maxDocIdCacheSize = 10000; + + private int maxDocsForInMemoryMerge = 10000; + + private int maxDocumentCacheSize = 100; + + private int maxIsCategoryCacheSize = -1; + + private int maxLinkAspectCacheSize = 10000; + + private int maxParentCacheSize = 10000; + + private int maxPathCacheSize = 10000; + + private int maxTypeCacheSize = 10000; + + private int mergerMaxMergeDocs = 1000000; + + private int mergerMergeFactor = 5; + + private int mergerMinMergeDocs = 1000; + + private int mergerTargetIndexCount = 5; + + private int mergerTargetOverlayCount = 5; + + private int termIndexInterval =IndexWriter.DEFAULT_TERM_INDEX_INTERVAL; + + private boolean useNioMemoryMapping = true; + + private int writerMaxMergeDocs = 1000000; + + private int writerMergeFactor = 5; + + private int writerMinMergeDocs = 1000; + + private boolean cacheEnabled = true; + /** * Private constructor for the singleton TODO: FIt in with IOC */ @@ -1462,6 +1500,197 @@ public abstract class AbstractLuceneIndexerAndSearcherFactory implements LuceneI defaultMLSearchAnalysisMode = mode; } + + public int getMaxDocIdCacheSize() + { + return maxDocIdCacheSize; + } + + public void setMaxDocIdCacheSize(int maxDocIdCacheSize) + { + this.maxDocIdCacheSize = maxDocIdCacheSize; + } + + public int getMaxDocsForInMemoryMerge() + { + return maxDocsForInMemoryMerge; + } + + public void setMaxDocsForInMemoryMerge(int maxDocsForInMemoryMerge) + { + this.maxDocsForInMemoryMerge = maxDocsForInMemoryMerge; + } + + public int getMaxDocumentCacheSize() + { + return maxDocumentCacheSize; + } + + public void setMaxDocumentCacheSize(int maxDocumentCacheSize) + { + this.maxDocumentCacheSize = maxDocumentCacheSize; + } + + public int getMaxIsCategoryCacheSize() + { + return maxIsCategoryCacheSize; + } + + public void setMaxIsCategoryCacheSize(int maxIsCategoryCacheSize) + { + this.maxIsCategoryCacheSize = maxIsCategoryCacheSize; + } + + public int getMaxLinkAspectCacheSize() + { + return maxLinkAspectCacheSize; + } + + public void setMaxLinkAspectCacheSize(int maxLinkAspectCacheSize) + { + this.maxLinkAspectCacheSize = maxLinkAspectCacheSize; + } + + public int getMaxParentCacheSize() + { + return maxParentCacheSize; + } + + public void setMaxParentCacheSize(int maxParentCacheSize) + { + this.maxParentCacheSize = maxParentCacheSize; + } + + public int getMaxPathCacheSize() + { + return maxPathCacheSize; + } + + public void setMaxPathCacheSize(int maxPathCacheSize) + { + this.maxPathCacheSize = maxPathCacheSize; + } + + public int getMaxTypeCacheSize() + { + return maxTypeCacheSize; + } + + public void setMaxTypeCacheSize(int maxTypeCacheSize) + { + this.maxTypeCacheSize = maxTypeCacheSize; + } + + public int getMergerMaxMergeDocs() + { + return mergerMaxMergeDocs; + } + + public void setMergerMaxMergeDocs(int mergerMaxMergeDocs) + { + this.mergerMaxMergeDocs = mergerMaxMergeDocs; + } + + public int getMergerMergeFactor() + { + return mergerMergeFactor; + } + + public void setMergerMergeFactor(int mergerMergeFactor) + { + this.mergerMergeFactor = mergerMergeFactor; + } + + public int getMergerMinMergeDocs() + { + return mergerMinMergeDocs; + } + + public void setMergerMinMergeDocs(int mergerMinMergeDocs) + { + this.mergerMinMergeDocs = mergerMinMergeDocs; + } + + public int getMergerTargetIndexCount() + { + return mergerTargetIndexCount; + } + + public void setMergerTargetIndexCount(int mergerTargetIndexCount) + { + this.mergerTargetIndexCount = mergerTargetIndexCount; + } + + public int getMergerTargetOverlayCount() + { + return mergerTargetOverlayCount; + } + + public void setMergerTargetOverlayCount(int mergerTargetOverlayCount) + { + this.mergerTargetOverlayCount = mergerTargetOverlayCount; + } + + public int getTermIndexInterval() + { + return termIndexInterval; + } + + public void setTermIndexInterval(int termIndexInterval) + { + this.termIndexInterval = termIndexInterval; + } + + public boolean getUseNioMemoryMapping() + { + return useNioMemoryMapping; + } + + public void setUseNioMemoryMapping(boolean useNioMemoryMapping) + { + this.useNioMemoryMapping = useNioMemoryMapping; + } + + public int getWriterMaxMergeDocs() + { + return writerMaxMergeDocs; + } + + public void setWriterMaxMergeDocs(int writerMaxMergeDocs) + { + this.writerMaxMergeDocs = writerMaxMergeDocs; + } + + public int getWriterMergeFactor() + { + return writerMergeFactor; + } + + public void setWriterMergeFactor(int writerMergeFactor) + { + this.writerMergeFactor = writerMergeFactor; + } + + public int getWriterMinMergeDocs() + { + return writerMinMergeDocs; + } + + public void setWriterMinMergeDocs(int writerMinMergeDocs) + { + this.writerMinMergeDocs = writerMinMergeDocs; + } + + public boolean isCacheEnabled() + { + return cacheEnabled; + } + + public void setCacheEnabled(boolean cacheEnabled) + { + this.cacheEnabled = cacheEnabled; + } + protected abstract List getAllStores(); public R doWithAllWriteLocks(WithAllWriteLocksWork lockWork) diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneConfig.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneConfig.java index 88b2684f81..28b8a8d663 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/LuceneConfig.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneConfig.java @@ -94,4 +94,118 @@ public interface LuceneConfig */ public BulkLoader getBulkLoader(); + /** + * Use the nio memory mapping (work arounf for bugs with some JVMs) + * @return + */ + public boolean getUseNioMemoryMapping(); + + /** + * Max doc number that will merged in memory (and not on disk) + * + * @return + */ + public int getMaxDocsForInMemoryMerge(); + + /** + * Lucene writer config + * @return + */ + public int getWriterMinMergeDocs(); + + /** + * Lucene writer config + * @return + */ + public int getWriterMergeFactor(); + + /** + * Lucene writer config + * @return + */ + public int getWriterMaxMergeDocs(); + + /** + * Lucene merger config + * @return + */ + public int getMergerMinMergeDocs(); + + /** + * Lucene merger config + * @return + */ + public int getMergerMergeFactor(); + + /** + * Lucene merger config + * @return + */ + public int getMergerMaxMergeDocs(); + + /** + * Target overlays (will apply deletions and create indexes if over this limit) + * @return + */ + public int getMergerTargetOverlayCount(); + + /** + * Target index count. Over this indexes will be merged together. + * @return + */ + public int getMergerTargetIndexCount(); + + /** + * Lucene term index interval + * @return + */ + public int getTermIndexInterval(); + + /** + * Is caching enabled for each index fragment? + * @return + */ + public boolean isCacheEnabled(); + + /** + * How many categories to cache (-ve => unbounded) + * @return + */ + public int getMaxIsCategoryCacheSize(); + + /** + * How many documents to cache (-ve => unbounded) + * @return + */ + public int getMaxDocumentCacheSize(); + + /** + * How many document ids to cache (-ve => unbounded) + * @return + */ + public int getMaxDocIdCacheSize(); + + /** + * How many paths to cache (-ve => unbounded) + * @return + */ + public int getMaxPathCacheSize(); + + /** + * How many types to cache (-ve => unbounded) + * @return + */ + public int getMaxTypeCacheSize(); + + /** + * How many parents to cache (-ve => unbounded) + * @return + */ + public int getMaxParentCacheSize(); + + /** + * How many link aspects to cache (-ve => unbounded) + * @return + */ + public int getMaxLinkAspectCacheSize(); } diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/CachingIndexReader.java b/source/java/org/alfresco/repo/search/impl/lucene/index/CachingIndexReader.java index 0ff42b01e7..70194d8c84 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/CachingIndexReader.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/CachingIndexReader.java @@ -35,6 +35,8 @@ public interface CachingIndexReader { public String getId(int n) throws IOException; + public String getPathLinkId(int n) throws IOException; + public String[] getIds(int n) throws IOException; public String getIsCategory(int n) throws IOException; diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java index ff74d91c0d..9fb8cda11f 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java @@ -132,7 +132,7 @@ public class IndexInfo /** * Use NIO memory mapping to wite the index control file. */ - private static final boolean useNIOMemoryMapping = true; + private static boolean useNIOMemoryMapping = true; /** * The default name for the file that holds the index information @@ -305,20 +305,14 @@ public class IndexInfo private int termIndexInterval = IndexWriter.DEFAULT_TERM_INDEX_INTERVAL; - /** - * Control if the cleaner thread is active - */ - - private boolean enableCleaner = true; - /** * Control if the merger thread is active */ - private boolean enableMerger = true; - private ThreadPoolExecutor threadPoolExecutor; + private LuceneConfig config; + static { // We do not require any of the lucene in-built locking. @@ -371,11 +365,22 @@ public class IndexInfo { super(); initialiseTransitions(); - + this.config = config; + if (config != null) { this.maxFieldLength = config.getIndexerMaxFieldLength(); this.threadPoolExecutor = config.getThreadPoolExecutor(); + IndexInfo.useNIOMemoryMapping = config.getUseNioMemoryMapping(); + this.maxDocsForInMemoryMerge = config.getMaxDocsForInMemoryMerge(); + this.writerMinMergeDocs = config.getWriterMinMergeDocs(); + this.writerMergeFactor = config.getWriterMergeFactor(); + this.writerMaxMergeDocs = config.getWriterMaxMergeDocs(); + this.mergerMinMergeDocs = config.getMergerMinMergeDocs(); + this.mergerMergeFactor = config.getMergerMergeFactor(); + this.mergerMaxMergeDocs = config.getMergerMaxMergeDocs(); + this.termIndexInterval = config.getTermIndexInterval(); + this.mergerTargetOverlays = config.getMergerTargetOverlayCount(); } else { @@ -474,7 +479,7 @@ public class IndexInfo writeStatus(); // The index exists and we should initialise the single reader - registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName())); + registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName(), entry.getDocumentCount())); } catch (IOException e) { @@ -548,7 +553,7 @@ public class IndexInfo s_logger.info("Resetting merge to committed " + entry); } entry.setStatus(TransactionStatus.COMMITTED); - registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName())); + registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName(), entry.getDocumentCount())); break; // Complete committing (which is post database // commit) @@ -559,12 +564,12 @@ public class IndexInfo s_logger.info("Committing " + entry); } entry.setStatus(TransactionStatus.COMMITTED); - registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName())); + registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName(), entry.getDocumentCount())); mainIndexReader = null; break; // States that require no action case COMMITTED: - registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName())); + registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName(), entry.getDocumentCount())); break; default: // nothing to do @@ -650,6 +655,10 @@ public class IndexInfo String id = file.getName(); if (!indexEntries.containsKey(id) && isGUID(id)) { + if (s_logger.isDebugEnabled()) + { + s_logger.debug("Deleting unused index directory " + id); + } deleteQueue.add(id); } } @@ -1048,7 +1057,7 @@ public class IndexInfo } return mainIndexReader; } - catch(RuntimeException e) + catch (RuntimeException e) { e.printStackTrace(); throw e; @@ -1147,7 +1156,7 @@ public class IndexInfo { reader = new MultiReader(new IndexReader[] { new FilterIndexReaderByStringId("main+id", mainIndexReader, deletions, deleteOnlyNodes), deltaReader }); } - reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(MAIN_READER + id, reader, false); + reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(MAIN_READER + id, reader, false, config); ReferenceCounting refCounting = (ReferenceCounting) reader; refCounting.incrementReferenceCount(); refCounting.setInvalidForReuse(); @@ -1427,7 +1436,8 @@ public class IndexInfo // Make sure we have set up the reader for the data // ... and close it so we do not up the ref count closeDelta(id); - tl.set(buildReferenceCountingIndexReader(id)); + IndexEntry entry = indexEntries.get(id); + tl.set(buildReferenceCountingIndexReader(id, entry.getDocumentCount())); } /** @@ -1538,7 +1548,8 @@ public class IndexInfo public void beforeWithReadLock(String id, Set toDelete, Set read) throws IOException { closeDelta(id); - tl.set(buildReferenceCountingIndexReader(id)); + IndexEntry entry = indexEntries.get(id); + tl.set(buildReferenceCountingIndexReader(id, entry.getDocumentCount())); } public void transition(String id, Set toDelete, Set read) throws IOException @@ -1825,7 +1836,7 @@ public class IndexInfo { reader = IndexReader.open(emptyIndex); } - reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(MAIN_READER, reader, false); + reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(MAIN_READER, reader, false, config); return reader; } @@ -1851,19 +1862,27 @@ public class IndexInfo referenceCountingReadOnlyIndexReaders.put(id, reader); } - private IndexReader buildReferenceCountingIndexReader(String id) throws IOException + private IndexReader buildReferenceCountingIndexReader(String id, long size) throws IOException { IndexReader reader; File location = new File(indexDirectory, id).getCanonicalFile(); if (IndexReader.indexExists(location)) { - reader = IndexReader.open(location); + if (size > config.getMaxDocsForInMemoryMerge()) + { + reader = IndexReader.open(location); + } + else + { + RAMDirectory rd = new RAMDirectory(location); + reader = IndexReader.open(rd); + } } else { reader = IndexReader.open(emptyIndex); } - reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(id, reader, true); + reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(id, reader, true, config); return reader; } @@ -2581,7 +2600,7 @@ public class IndexInfo throw new IllegalStateException(); } } - + private synchronized void rescheduleRecovery() { switch (scheduledState) @@ -2869,7 +2888,7 @@ public class IndexInfo // Check we have a reader registered if (referenceCountingReadOnlyIndexReaders.get(entry.getName()) == null) { - registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName())); + registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName(), entry.getDocumentCount())); } } @@ -3126,7 +3145,7 @@ public class IndexInfo for (String id : invalidIndexes) { - IndexReader reader = buildReferenceCountingIndexReader(id); + IndexReader reader = buildReferenceCountingIndexReader(id, newIndexCounts.get(id)); newReaders.put(id, reader); } @@ -3325,13 +3344,15 @@ public class IndexInfo String mergeTargetId = null; + long docCount = 0; + if (toMerge.size() > 0) { int count = 0; IndexReader[] readers = new IndexReader[toMerge.size() - 1]; RAMDirectory ramDirectory = null; IndexWriter writer = null; - long docCount = 0; + File outputLocation = null; for (IndexEntry entry : toMerge.values()) { @@ -3409,7 +3430,7 @@ public class IndexInfo getReadLock(); try { - newReader = buildReferenceCountingIndexReader(mergeTargetId); + newReader = buildReferenceCountingIndexReader(mergeTargetId, docCount); } finally { @@ -3598,136 +3619,6 @@ public class IndexInfo return builder.toString(); } - public boolean isEnableCleanerThread() - { - return enableCleaner; - } - - public void setEnableCleanerThread(boolean enableCleaner) - { - this.enableCleaner = enableCleaner; - } - - public boolean isEnableMerger() - { - return enableMerger; - } - - public void setEnableMerger(boolean enableMerger) - { - this.enableMerger = enableMerger; - } - - public boolean isIndexIsShared() - { - return indexIsShared; - } - - public void setIndexIsShared(boolean indexIsShared) - { - this.indexIsShared = indexIsShared; - } - - public int getMaxDocsForInMemoryMerge() - { - return maxDocsForInMemoryMerge; - } - - public void setMaxDocsForInMemoryMerge(int maxDocsForInMemoryMerge) - { - this.maxDocsForInMemoryMerge = maxDocsForInMemoryMerge; - } - - public int getMergerMaxMergeDocs() - { - return mergerMaxMergeDocs; - } - - public void setMergerMaxMergeDocs(int mergerMaxMergeDocs) - { - this.mergerMaxMergeDocs = mergerMaxMergeDocs; - } - - public int getMergerMergeFactor() - { - return mergerMergeFactor; - } - - public void setMergerMergeFactor(int mergerMergeFactor) - { - this.mergerMergeFactor = mergerMergeFactor; - } - - public int getMergerMinMergeDocs() - { - return mergerMinMergeDocs; - } - - public void setMergerMinMergeDocs(int mergerMinMergeDocs) - { - this.mergerMinMergeDocs = mergerMinMergeDocs; - } - - public int getMergerTargetOverlays() - { - return mergerTargetOverlays; - } - - public void setMergerTargetOverlays(int mergerTargetOverlays) - { - this.mergerTargetOverlays = mergerTargetOverlays; - } - - public boolean isMergerUseCompoundFile() - { - return mergerUseCompoundFile; - } - - public void setMergerUseCompoundFile(boolean mergerUseCompoundFile) - { - this.mergerUseCompoundFile = mergerUseCompoundFile; - } - - public int getWriterMaxMergeDocs() - { - return writerMaxMergeDocs; - } - - public void setWriterMaxMergeDocs(int writerMaxMergeDocs) - { - this.writerMaxMergeDocs = writerMaxMergeDocs; - } - - public int getWriterMergeFactor() - { - return writerMergeFactor; - } - - public void setWriterMergeFactor(int writerMergeFactor) - { - this.writerMergeFactor = writerMergeFactor; - } - - public int getWriterMinMergeDocs() - { - return writerMinMergeDocs; - } - - public void setWriterMinMergeDocs(int writerMinMergeDocs) - { - this.writerMinMergeDocs = writerMinMergeDocs; - } - - public boolean isWriterUseCompoundFile() - { - return writerUseCompoundFile; - } - - public void setWriterUseCompoundFile(boolean writerUseCompoundFile) - { - this.writerUseCompoundFile = writerUseCompoundFile; - } - private boolean isGUID(String guid) { try diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java b/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java index 42ac9d0de8..ed414bcf83 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.BitSet; import java.util.Collection; import java.util.HashMap; @@ -38,11 +39,13 @@ import java.util.ListIterator; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import net.sf.ehcache.CacheManager; +import org.alfresco.repo.search.impl.lucene.LuceneConfig; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.document.Document; @@ -60,9 +63,9 @@ public class ReferenceCountingReadOnlyIndexReaderFactory private static HashMap log = new HashMap(); - public static IndexReader createReader(String id, IndexReader indexReader, boolean enableCaching) + public static IndexReader createReader(String id, IndexReader indexReader, boolean enableCaching, LuceneConfig config) { - ReferenceCountingReadOnlyIndexReader rc = new ReferenceCountingReadOnlyIndexReader(id, indexReader, enableCaching); + ReferenceCountingReadOnlyIndexReader rc = new ReferenceCountingReadOnlyIndexReader(id, indexReader, enableCaching, config); if (s_logger.isDebugEnabled()) { if (log.containsKey(id)) @@ -108,26 +111,33 @@ public class ReferenceCountingReadOnlyIndexReaderFactory boolean closed = false; - // CacheManager cacheManager; - - ConcurrentMap documentCache = concurrentMap(new LinkedHashMap()); // new - - ConcurrentSet documentKeys = (ConcurrentSet) documentCache.keySet(); - - ConcurrentMap idCache = concurrentMap(new LinkedHashMap()); // new - - ConcurrentSet idKeys = (ConcurrentSet) idCache.keySet(); - ConcurrentHashMap isCategory = new ConcurrentHashMap(); + ConcurrentHashMap>> documentCache = new ConcurrentHashMap>>(); + + ConcurrentHashMap>> idCache = new ConcurrentHashMap>>(); + + ConcurrentHashMap> pathCache = new ConcurrentHashMap>(); + + ConcurrentHashMap> typeCache = new ConcurrentHashMap>(); + + ConcurrentHashMap>> parentCache = new ConcurrentHashMap>>(); + + ConcurrentHashMap>> linkAspectCache = new ConcurrentHashMap>>(); + boolean enableCaching; - ReferenceCountingReadOnlyIndexReader(String id, IndexReader indexReader, boolean enableCaching) + private LuceneConfig config; + + ReferenceCountingReadOnlyIndexReader(String id, IndexReader indexReader, boolean enableCaching, LuceneConfig config) { super(indexReader); this.id = id; - this.enableCaching = enableCaching; - + if(enableCaching && (config != null)) + { + this.enableCaching = config.isCacheEnabled(); + } + this.config = config; } public synchronized void incrementReferenceCount() @@ -231,44 +241,131 @@ public class ReferenceCountingReadOnlyIndexReaderFactory throw new UnsupportedOperationException("Delete is not supported by read only index readers"); } + private interface Accessor + { + T get(int n, FieldSelector fieldSelector) throws IOException; + } + + private class ListFieldAccessor implements Accessor> + { + + public List get(int n, FieldSelector fieldSelector) throws IOException + { + Document document = ReferenceCountingReadOnlyIndexReader.super.document(n, fieldSelector); + List fields = (List) document.getFields(); + ArrayList cacheable = new ArrayList(fields.size()); + cacheable.addAll(fields); + return cacheable; + } + + } + + private class MultipleValueFieldAccessor implements Accessor> + { + String fieldName; + + MultipleValueFieldAccessor(String fieldName) + { + this.fieldName = fieldName; + } + + public List get(int n, FieldSelector fieldSelector) throws IOException + { + Document document = ReferenceCountingReadOnlyIndexReader.super.document(n, fieldSelector); + Field[] fields = document.getFields(fieldName); + ArrayList cacheable = new ArrayList(fields.length); + for (Field field : fields) + { + cacheable.add(field); + } + return cacheable; + } + + } + + private class SingleValueFieldAccessor implements Accessor + { + String fieldName; + + SingleValueFieldAccessor(String fieldName) + { + this.fieldName = fieldName; + } + + public Field get(int n, FieldSelector fieldSelector) throws IOException + { + return new Field(fieldName, getStringValue(n, fieldName), Store.NO, Index.UN_TOKENIZED); + } + + } + + private final ListFieldAccessor LIST_FIELD_ACCESSOR = new ListFieldAccessor(); + + private final MultipleValueFieldAccessor MV_ID_FIELD_ACCESSOR = new MultipleValueFieldAccessor("ID"); + + private final SingleValueFieldAccessor SV_TYPE_FIELD_ACCESSOR = new SingleValueFieldAccessor("TYPE"); + + private final SingleValueFieldAccessor SV_PATH_FIELD_ACCESSOR = new SingleValueFieldAccessor("PATH"); + + private final MultipleValueFieldAccessor MV_PARENT_FIELD_ACCESSOR = new MultipleValueFieldAccessor("PARENT"); + + private final MultipleValueFieldAccessor MV_LINKASPECT_FIELD_ACCESSOR = new MultipleValueFieldAccessor("LINKASPECT"); + + private T manageCache(ConcurrentHashMap> cache, Accessor accessor, int n, FieldSelector fieldSelector, int limit) throws IOException + { + Integer key = Integer.valueOf(n); + WithUseCount value = cache.get(key); + if (value == null) + { + T made = accessor.get(n, fieldSelector); + value = new WithUseCount(made, n); + cache.put(key, value); + + // resize + + if (limit >= 0) + { + if (cache.size() >= limit) + { + HashMap> keep = new HashMap>(); + WithUseCount[] existing = new WithUseCount[0]; + synchronized (cache) + { + existing = cache.values().toArray(existing); + cache.clear(); + } + Arrays.sort(existing); + + for (WithUseCount current : existing) + { + keep.put(Integer.valueOf(current.doc), current); + if ((current.count.get() == 0) || (keep.size() > (limit / 4))) + { + break; + } + } + keep.put(key, value); + cache.putAll(keep); + } + } + } + else + { + value.count.getAndIncrement(); + } + return value.object; + } + + @SuppressWarnings("unchecked") public Document document(int n, FieldSelector fieldSelector) throws IOException { if ((fieldSelector == null) && enableCaching) { - Integer key = Integer.valueOf(n); - DocumentWithInstanceCount document = documentCache.get(key); - if (document == null) - { + List listOfFields = manageCache(documentCache, LIST_FIELD_ACCESSOR, n, fieldSelector, config.getMaxDocumentCacheSize()); + Document document = new Document(); + document.getFields().addAll(listOfFields); + return document; - document = new DocumentWithInstanceCount(super.document(n, fieldSelector)); - if (document.instance < 0) - { - DocumentWithInstanceCount.instanceCount = 0; - new DocumentWithInstanceCount(document.document); - documentCache.clear(); - } - documentCache.put(key, document); - - if (documentCache.size() > 100) - { - documentCache.resize(50); - } - } - else - { - if (document.instance < DocumentWithInstanceCount.instanceCount - 50) - { - document = new DocumentWithInstanceCount(document.document); - if (document.instance < 0) - { - DocumentWithInstanceCount.instanceCount = 0; - new DocumentWithInstanceCount(document.document); - documentCache.clear(); - } - documentCache.replace(key, new DocumentWithInstanceCount(document.document)); - } - } - return document.document; } else { @@ -276,44 +373,13 @@ public class ReferenceCountingReadOnlyIndexReaderFactory { SingleFieldSelector sfs = (SingleFieldSelector) fieldSelector; - if (sfs.field.equals("ID") && sfs.last) + if (sfs.field.equals("ID") && !sfs.last) { - Integer key = Integer.valueOf(n); - IdWithInstanceCount id = idCache.get(key); - if (id == null) - { - - id = new IdWithInstanceCount(getLastStringValue(n, "ID")); - if (id.instance < 0) - { - IdWithInstanceCount.instanceCount = 0; - new IdWithInstanceCount(id.id); - idCache.clear(); - } - idCache.put(key, id); - - if (idCache.size() > 10000) - { - idCache.resize(5000); - } - } - else - { - if (id.instance < IdWithInstanceCount.instanceCount - 5000) - { - id = new IdWithInstanceCount(id.id); - if (id.instance < 0) - { - IdWithInstanceCount.instanceCount = 0; - new IdWithInstanceCount(id.id); - idCache.clear(); - } - idCache.replace(key, new IdWithInstanceCount(id.id)); - } - } + List idFields = manageCache(idCache, MV_ID_FIELD_ACCESSOR, n, fieldSelector, config.getMaxDocIdCacheSize()); Document d = new Document(); - d.add(new Field("ID", id.id, Store.NO, Index.UN_TOKENIZED)); + d.getFields().addAll(idFields); return d; + } if (sfs.field.equals("ISCATEGORY") && sfs.last) { @@ -331,6 +397,38 @@ public class ReferenceCountingReadOnlyIndexReaderFactory } return d; } + if (sfs.field.equals("PATH") && sfs.last) + { + Field pathField = manageCache(pathCache, SV_PATH_FIELD_ACCESSOR, n, fieldSelector, config.getMaxPathCacheSize()); + Document d = new Document(); + d.add(pathField); + return d; + + } + if (sfs.field.equals("TYPE") && sfs.last) + { + Field typeField = manageCache(typeCache, SV_TYPE_FIELD_ACCESSOR, n, fieldSelector, config.getMaxTypeCacheSize()); + Document d = new Document(); + d.add(typeField); + return d; + + } + if (sfs.field.equals("PARENT") && !sfs.last) + { + List listOfFields = manageCache(parentCache, MV_PARENT_FIELD_ACCESSOR, n, fieldSelector, config.getMaxParentCacheSize()); + Document document = new Document(); + document.getFields().addAll(listOfFields); + return document; + + } + if (sfs.field.equals("LINKASPECT") && !sfs.last) + { + List listOfFields = manageCache(linkAspectCache, MV_LINKASPECT_FIELD_ACCESSOR, n, fieldSelector, config.getMaxLinkAspectCacheSize()); + Document document = new Document(); + document.getFields().addAll(listOfFields); + return document; + + } } } @@ -354,6 +452,14 @@ public class ReferenceCountingReadOnlyIndexReaderFactory return d.getField("ID").stringValue(); } + public String getPathLinkId(int n) throws IOException + { + Document document = document(n, new SingleFieldSelector("ID", true)); + Field[] fields = document.getFields("ID"); + Field field = fields[fields.length - 1]; + return (field == null) ? null : field.stringValue(); + } + public String[] getIds(int n) throws IOException { return getStringValues(n, "ID"); @@ -361,22 +467,32 @@ public class ReferenceCountingReadOnlyIndexReaderFactory public String[] getLinkAspects(int n) throws IOException { - return getStringValues(n, "LINKASPECT"); + // return getStringValues(n, "LINKASPECT"); + Document d = document(n, new SingleFieldSelector("LINKASPECT", false)); + return d.getValues("LINKASPECT"); } public String[] getParents(int n) throws IOException { - return getStringValues(n, "PARENT"); + // return getStringValues(n, "PARENT"); + Document d = document(n, new SingleFieldSelector("PARENT", false)); + return d.getValues("PARENT"); } public String getPath(int n) throws IOException { - return getStringValue(n, "PATH"); + // return getStringValue(n, "PATH"); + Document d = document(n, new SingleFieldSelector("PATH", true)); + Field f = d.getField("PATH"); + return f == null ? null : f.stringValue(); } public String getType(int n) throws IOException { - return getStringValue(n, "TYPE"); + // return getStringValue(n, "TYPE"); + Document d = document(n, new SingleFieldSelector("TYPE", true)); + Field f = d.getField("TYPE"); + return f == null ? null : f.stringValue(); } public String getIsCategory(int n) throws IOException @@ -386,13 +502,13 @@ public class ReferenceCountingReadOnlyIndexReaderFactory return f == null ? null : f.stringValue(); } - private String getLastStringValue(int n, String fieldName) throws IOException - { - Document document = document(n); - Field[] fields = document.getFields(fieldName); - Field field = fields[fields.length - 1]; - return (field == null) ? null : field.stringValue(); - } + // private String getLastStringValue(int n, String fieldName) throws IOException + // { + // Document document = document(n); + // Field[] fields = document.getFields(fieldName); + // Field field = fields[fields.length - 1]; + // return (field == null) ? null : field.stringValue(); + // } private String getStringValue(int n, String fieldName) throws IOException { @@ -401,789 +517,43 @@ public class ReferenceCountingReadOnlyIndexReaderFactory return (field == null) ? null : field.stringValue(); } + @SuppressWarnings("unchecked") private String[] getStringValues(int n, String fieldName) throws IOException { Document document = document(n); - Field[] fields = document.getFields(fieldName); - if (fields != null) + ArrayList fields = new ArrayList(); + ArrayList answer = new ArrayList(2); + fields.addAll((List) document.getFields()); + + for (Field field : fields) { - String[] answer = new String[fields.length]; - int i = 0; - for (Field field : fields) + if (field.name().equals(fieldName)) { - answer[i++] = (field == null) ? null : field.stringValue(); - } - return answer; - } - else - { - return null; - } - } - } - - public static ConcurrentMap concurrentMap(Map m) - { - return new ConcurrentMap(m); - } - - private static class ConcurrentMap implements Map, Serializable - { - private final Map m; // Backing Map - - final private ReadWriteLock readWriteLock; - - ConcurrentMap(Map m) - { - if (m == null) - throw new NullPointerException(); - this.m = m; - this.readWriteLock = new ReentrantReadWriteLock(); - } - - ConcurrentMap(Map m, ReadWriteLock readWriteLock) - { - this.m = m; - this.readWriteLock = readWriteLock; - } - - public int size() - { - readWriteLock.readLock().lock(); - try - { - return m.size(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public boolean isEmpty() - { - readWriteLock.readLock().lock(); - try - { - return m.isEmpty(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public boolean containsKey(Object key) - { - readWriteLock.readLock().lock(); - try - { - return m.containsKey(key); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public boolean containsValue(Object value) - { - readWriteLock.readLock().lock(); - try - { - return m.containsValue(value); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public V get(Object key) - { - readWriteLock.readLock().lock(); - try - { - return m.get(key); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public V put(K key, V value) - { - - readWriteLock.writeLock().lock(); - try - { - return m.put(key, value); - } - finally - { - readWriteLock.writeLock().unlock(); - } - } - - public V remove(Object key) - { - readWriteLock.writeLock().lock(); - try - { - return m.remove(key); - } - finally - { - readWriteLock.writeLock().unlock(); - } - } - - public void putAll(Map map) - { - readWriteLock.writeLock().lock(); - try - { - m.putAll(map); - } - finally - { - readWriteLock.writeLock().unlock(); - } - } - - public void clear() - { - readWriteLock.writeLock().lock(); - try - { - m.clear(); - } - finally - { - readWriteLock.writeLock().unlock(); - } - } - - private transient Set keySet = null; - - private transient Set> entrySet = null; - - private transient Collection values = null; - - public Set keySet() - { - - readWriteLock.readLock().lock(); - try - { - if (keySet == null) - keySet = new ConcurrentSet(m.keySet(), readWriteLock); - return keySet; - } - finally - { - readWriteLock.readLock().unlock(); - } - - } - - public Set> entrySet() - { - readWriteLock.readLock().lock(); - try - { - if (entrySet == null) - entrySet = new ConcurrentSet>(m.entrySet(), readWriteLock); - return entrySet; - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public Collection values() - { - readWriteLock.readLock().lock(); - try - { - if (values == null) - values = new ConcurrentCollection(m.values(), readWriteLock); - return values; - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public boolean equals(Object o) - { - readWriteLock.readLock().lock(); - try - { - return m.equals(o); - } - finally - { - readWriteLock.readLock().unlock(); - } - - } - - public int hashCode() - { - readWriteLock.readLock().lock(); - try - { - return m.hashCode(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public String toString() - { - readWriteLock.readLock().lock(); - try - { - return m.toString(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - private void writeObject(ObjectOutputStream s) throws IOException - { - readWriteLock.readLock().lock(); - try - { - s.defaultWriteObject(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public void resize(int size) - { - - ArrayList toRemove = null; - readWriteLock.writeLock().lock(); - try - { - int excess = m.size() - size; - toRemove = new ArrayList(excess); - if (excess > 0) - { - - Iterator it = keySet.iterator(); - while (toRemove.size() < excess) - { - K key = it.next(); - toRemove.add(key); - } - } - - } - finally - { - readWriteLock.writeLock().unlock(); - } - - if ((toRemove != null) && (toRemove.size() > 0)) - { - readWriteLock.writeLock().lock(); - try - { - for (K key : toRemove) - { - m.remove(key); - } - } - finally - { - readWriteLock.writeLock().unlock(); + answer.add(field.stringValue()); } } - } - public void replace(K key, V value) - { - readWriteLock.writeLock().lock(); - try - { - m.remove(key); - m.put(key, value); - } - finally - { - readWriteLock.writeLock().unlock(); - } + return answer.toArray(new String[answer.size()]); } } - /** - * @serial include - */ - static class ConcurrentCollection implements Collection, Serializable + static class WithUseCount implements Comparable> { - final Collection c; // Backing Collection + AtomicInteger count = new AtomicInteger(0); - final ReadWriteLock readWriteLock; + T object; - ConcurrentCollection(Collection c) + int doc; + + WithUseCount(T object, int doc) { - if (c == null) - throw new NullPointerException(); - this.c = c; - this.readWriteLock = new ReentrantReadWriteLock(); + this.object = object; + this.doc = doc; } - ConcurrentCollection(Collection c, ReadWriteLock readWriteLock) + public int compareTo(WithUseCount other) { - this.c = c; - this.readWriteLock = readWriteLock; - } - - public int size() - { - readWriteLock.readLock().lock(); - try - { - return c.size(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public boolean isEmpty() - { - readWriteLock.readLock().lock(); - try - { - return c.isEmpty(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public boolean contains(Object o) - { - readWriteLock.readLock().lock(); - try - { - return c.contains(o); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public Object[] toArray() - { - readWriteLock.readLock().lock(); - try - { - return c.toArray(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public T[] toArray(T[] a) - { - readWriteLock.readLock().lock(); - try - { - return c.toArray(a); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public Iterator iterator() - { - return c.iterator(); // Must be manually synched by user! - } - - public boolean add(E e) - { - readWriteLock.writeLock().lock(); - try - { - return c.add(e); - } - finally - { - readWriteLock.writeLock().unlock(); - } - - } - - public boolean remove(Object o) - { - readWriteLock.writeLock().lock(); - try - { - return c.remove(o); - } - finally - { - readWriteLock.writeLock().unlock(); - } - } - - public boolean containsAll(Collection coll) - { - readWriteLock.readLock().lock(); - try - { - return c.containsAll(coll); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public boolean addAll(Collection coll) - { - readWriteLock.writeLock().lock(); - try - { - return c.addAll(coll); - } - finally - { - readWriteLock.writeLock().unlock(); - } - } - - public boolean removeAll(Collection coll) - { - readWriteLock.writeLock().lock(); - try - { - return c.removeAll(coll); - } - finally - { - readWriteLock.writeLock().unlock(); - } - } - - public boolean retainAll(Collection coll) - { - readWriteLock.writeLock().lock(); - try - { - return c.retainAll(coll); - } - finally - { - readWriteLock.writeLock().unlock(); - } - - } - - public void clear() - { - readWriteLock.writeLock().lock(); - try - { - c.clear(); - } - finally - { - readWriteLock.writeLock().unlock(); - } - } - - public String toString() - { - readWriteLock.readLock().lock(); - try - { - return c.toString(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - private void writeObject(ObjectOutputStream s) throws IOException - { - readWriteLock.readLock().lock(); - try - { - s.defaultWriteObject(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - } - - /** - * @serial include - */ - static class ConcurrentSet extends ConcurrentCollection implements Set - { - ConcurrentSet(Set s) - { - super(s); - } - - ConcurrentSet(Set s, ReadWriteLock readWriteLock) - { - super(s, readWriteLock); - } - - public boolean equals(Object o) - { - readWriteLock.readLock().lock(); - try - { - return c.equals(o); - } - finally - { - readWriteLock.readLock().unlock(); - } - - } - - public int hashCode() - { - readWriteLock.readLock().lock(); - try - { - return c.hashCode(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - } - - static class ConcurrentList extends ConcurrentCollection implements List - { - static final long serialVersionUID = -7754090372962971524L; - - final List list; - - ConcurrentList(List list) - { - super(list); - this.list = list; - } - - ConcurrentList(List list, ReadWriteLock readWriteLock) - { - super(list, readWriteLock); - this.list = list; - } - - public boolean equals(Object o) - { - readWriteLock.readLock().lock(); - try - { - return c.equals(o); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public int hashCode() - { - readWriteLock.readLock().lock(); - try - { - return c.hashCode(); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public E get(int index) - { - readWriteLock.readLock().lock(); - try - { - return list.get(index); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public E set(int index, E element) - { - readWriteLock.readLock().lock(); - try - { - return list.set(index, element); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public void add(int index, E element) - { - readWriteLock.readLock().lock(); - try - { - list.add(index, element); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public E remove(int index) - { - readWriteLock.readLock().lock(); - try - { - return list.remove(index); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public int indexOf(Object o) - { - readWriteLock.readLock().lock(); - try - { - return list.indexOf(o); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public int lastIndexOf(Object o) - { - readWriteLock.readLock().lock(); - try - { - return list.lastIndexOf(o); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public boolean addAll(int index, Collection c) - { - readWriteLock.readLock().lock(); - try - { - return list.addAll(index, c); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - public ListIterator listIterator() - { - return list.listIterator(); // Must be manually synched by user - } - - public ListIterator listIterator(int index) - { - return list.listIterator(index); // Must be manually synched by user - } - - public List subList(int fromIndex, int toIndex) - { - readWriteLock.readLock().lock(); - try - { - return new ConcurrentList(list.subList(fromIndex, toIndex), readWriteLock); - } - finally - { - readWriteLock.readLock().unlock(); - } - } - - /** - * SynchronizedRandomAccessList instances are serialized as SynchronizedList instances to allow them to be - * deserialized in pre-1.4 JREs (which do not have SynchronizedRandomAccessList). This method inverts the - * transformation. As a beneficial side-effect, it also grafts the RandomAccess marker onto SynchronizedList - * instances that were serialized in pre-1.4 JREs. Note: Unfortunately, SynchronizedRandomAccessList instances - * serialized in 1.4.1 and deserialized in 1.4 will become SynchronizedList instances, as this method was - * missing in 1.4. - */ - // private Object readResolve() - // { - // return (list instanceof RandomAccess ? new SynchronizedRandomAccessList(list) : this); - // } - } - - static class DocumentWithInstanceCount - { - volatile static int instanceCount = 0; - - int instance; - - Document document; - - DocumentWithInstanceCount(Document document) - { - this.document = document; - instance = instanceCount++; - } - } - - static class IdWithInstanceCount - { - volatile static int instanceCount = 0; - - int instance; - - String id; - - IdWithInstanceCount(String id) - { - this.id = id; - instance = instanceCount++; + return other.count.get() - this.count.get(); } } diff --git a/source/java/org/alfresco/repo/search/impl/lucene/query/LeafScorer.java b/source/java/org/alfresco/repo/search/impl/lucene/query/LeafScorer.java index 1dad04db8f..8458b57fbc 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/query/LeafScorer.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/query/LeafScorer.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -35,12 +36,15 @@ import java.util.List; import org.alfresco.model.ContentModel; import org.alfresco.repo.search.SearcherException; import org.alfresco.repo.search.impl.lucene.index.CachingIndexReader; +import org.alfresco.repo.search.impl.lucene.index.IndexInfo; import org.alfresco.service.cmr.dictionary.AspectDefinition; import org.alfresco.service.cmr.dictionary.DataTypeDefinition; import org.alfresco.service.cmr.dictionary.DictionaryService; import org.alfresco.service.cmr.dictionary.PropertyDefinition; import org.alfresco.service.cmr.dictionary.TypeDefinition; import org.alfresco.service.namespace.QName; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; @@ -58,6 +62,11 @@ import org.apache.lucene.search.Weight; */ public class LeafScorer extends Scorer { + /** + * The logger. + */ + private static Log s_logger = LogFactory.getLog(IndexInfo.class); + static class Counter { int count = 0; @@ -116,7 +125,7 @@ public class LeafScorer extends Scorer private int[] cats; - private TermPositions tp; + // private TermPositions tp; /** * Constructor - should use an arg object ... @@ -143,7 +152,7 @@ public class LeafScorer extends Scorer this.containerScorer = containerScorer; this.sfps = sfps; this.allNodes = allNodes; - this.tp = tp; + // this.tp = tp; if (selfIds == null) { this.selfIds = new HashMap(); @@ -169,12 +178,12 @@ public class LeafScorer extends Scorer } - private String getId(IndexReader reader, int n) throws IOException + private String getPathLinkId(IndexReader reader, int n) throws IOException { if (reader instanceof CachingIndexReader) { CachingIndexReader cachingIndexReader = (CachingIndexReader) reader; - return cachingIndexReader.getId(n); + return cachingIndexReader.getPathLinkId(n); } else { @@ -221,7 +230,7 @@ public class LeafScorer extends Scorer return (path == null) ? null : path.stringValue(); } } - + private String getType(IndexReader reader, int n) throws IOException { if (reader instanceof CachingIndexReader) @@ -236,7 +245,7 @@ public class LeafScorer extends Scorer return (path == null) ? null : path.stringValue(); } } - + private String[] getParents(IndexReader reader, int n) throws IOException { if (reader instanceof CachingIndexReader) @@ -264,7 +273,7 @@ public class LeafScorer extends Scorer } } } - + private String[] getlinkAspects(IndexReader reader, int n) throws IOException { if (reader instanceof CachingIndexReader) @@ -292,8 +301,6 @@ public class LeafScorer extends Scorer } } } - - private void initialise() throws IOException { @@ -304,7 +311,7 @@ public class LeafScorer extends Scorer { int doc = containerScorer.doc(); - String id = getId(reader, doc); + String id = getPathLinkId(reader, doc); Counter counter = parentIds.get(id); if (counter == null) { @@ -348,7 +355,7 @@ public class LeafScorer extends Scorer while (level0.next()) { int doc = level0.doc(); - String id = getId(reader, doc); + String id = getPathLinkId(reader, doc); if (id != null) { Counter counter = parentIds.get(id); @@ -374,7 +381,10 @@ public class LeafScorer extends Scorer } else if (parentIds.size() == 0) { - throw new SearcherException("Index has no root node. Check that the correct index locations are being used."); + if (s_logger.isWarnEnabled()) + { + s_logger.warn("Index has no root node. Check that the correct index locations are being used."); + } } } @@ -382,10 +392,14 @@ public class LeafScorer extends Scorer { int position = 0; parents = new int[10000]; - for (String parent : parentIds.keySet()) + ArrayList ordered = new ArrayList(parentIds.size()); + ordered.addAll(parentIds.keySet()); + Collections.sort(ordered); + for (String parent : ordered) { Counter counter = parentIds.get(parent); - tp.seek(new Term("PARENT", parent)); + // tp.seek(new Term("PARENT", parent)); + TermPositions tp = reader.termPositions(new Term("PARENT", parent)); while (tp.next()) { for (int i = 0, l = tp.freq(); i < l; i++) @@ -412,9 +426,13 @@ public class LeafScorer extends Scorer position = 0; self = new int[10000]; - for (String id : selfIds.keySet()) + ordered = new ArrayList(selfIds.size()); + ordered.addAll(selfIds.keySet()); + Collections.sort(ordered); + for (String id : ordered) { - tp.seek(new Term("ID", id)); + // tp.seek(new Term("ID", id)); + TermPositions tp = reader.termPositions(new Term("ID", id)); while (tp.next()) { Counter counter = selfIds.get(id); @@ -438,7 +456,10 @@ public class LeafScorer extends Scorer position = 0; cats = new int[10000]; - for (String catid : categories.keySet()) + ordered = new ArrayList(categories.size()); + ordered.addAll(categories.keySet()); + Collections.sort(ordered); + for (String catid : ordered) { for (QName apsectQName : dictionaryService.getAllAspects()) { @@ -449,7 +470,8 @@ public class LeafScorer extends Scorer { if (propDef.getDataType().getName().equals(DataTypeDefinition.CATEGORY)) { - tp.seek(new Term("@" + propDef.getName().toString(), catid)); + // tp.seek(new Term("@" + propDef.getName().toString(), catid)); + TermPositions tp = reader.termPositions(new Term("@" + propDef.getName().toString(), catid)); while (tp.next()) { for (int i = 0, l = tp.freq(); i < l; i++) @@ -819,9 +841,14 @@ public class LeafScorer extends Scorer } } - //Document doc = reader.document(doc()); + // Document doc = reader.document(doc()); String[] parentFields = getParents(reader, doc()); - String[] linkFields = getlinkAspects(reader, doc()); + + String[] linkFields = null; + if (categories.size() > 0) + { + linkFields = getlinkAspects(reader, doc()); + } String parentID = null; String linkAspect = null; @@ -846,7 +873,7 @@ public class LeafScorer extends Scorer { return; } - String id = getId(reader, doc()); + String id = getPathLinkId(reader, doc()); StructuredFieldPosition last = sfps[sfps.length - 1]; if ((last.linkSelf() && selfIds.containsKey(id))) {