From a41642494bae27864c20980b021cc0c8b15c5f74 Mon Sep 17 00:00:00 2001 From: Alan Davis Date: Fri, 16 May 2014 16:31:42 +0000 Subject: [PATCH] Merged HEAD-BUG-FIX (4.3/Cloud) to HEAD (4.3/Cloud) 68532: Merged V4.2-BUG-FIX (4.2.3) to HEAD-BUG-FIX (4.3/Cloud) 68305: Merged V4.1-BUG-FIX (4.1.9) to V4.2-BUG-FIX (4.2.3) 68264: MNT-11343: Merged V4.1.6 (4.1.6.14) to V4.1-BUG-FIX (4.1.9) << There are really only 3 commits as follows. In addition there are a number of reverts and reverts of reverts: 59852+6039-60386, 60679-60680, 64213-64219, 65093-65103, 65450-65451 >> - 60675: MNT-10017: SOLR: Long reindex times when deleting or changing sites env with 15K sites New ability has been added to configure types which do not support paths calculation for indexing. New option has been added to enable or disable filtering types. Paths are also ignored for 'cm:person' and 'app:configurations' subtypes. Also parent associations are ignored for 'cm:person' and 'app:configurations' types and their subtypes: - 'search.solrTrackingSupport.ignorePathsForSpecificTypes' (default value is 'false'; usual behavior); - the configuration for the types supports full and prefixed QNames - 63983: MNT-10017: SOLR: Long reindex times when deleting or changing sites env with 15K sites Aux re-index logic has been modified to skip metadata pulling for types not matching type constraints. Set of types is configurable. 'SOLRTrackingComponentImpl.getNodesMetadata()' has been modified to filter children not matching type constraints. Additionally, logic for calculating category paths has been modified to take into account the type constraints - 65017: MNT-10017: SOLR: Long reindex times when deleting or changing sites env with 15K sites Nodes filtering (those do not need in cascade update for their child aux docs) has been improved. Ids of such docs are searched at start of cascade by configured types. Metadata of child are not pulled if result docs set contains id of the child. Node metadata may contain DB ids of child associations in the same time. Cache with doc id by DB id mapping doesn't exist in core tracker. Hence, these child associations can be filtered only using query by DB id to get doc id git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@70411 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261 --- .../Search/common-search-context.xml | 10 + .../Search/lucene/common-search.properties | 2 + .../Search/noindex/common-search.properties | 2 + .../Search/solr/common-search.properties | 2 + .../repo/solr/SOLRTrackingComponentImpl.java | 171 +++++++++++++++--- 5 files changed, 166 insertions(+), 21 deletions(-) diff --git a/config/alfresco/subsystems/Search/common-search-context.xml b/config/alfresco/subsystems/Search/common-search-context.xml index c15ab8bfa7..7ebff1a2fd 100644 --- a/config/alfresco/subsystems/Search/common-search-context.xml +++ b/config/alfresco/subsystems/Search/common-search-context.xml @@ -11,6 +11,7 @@ + @@ -19,6 +20,15 @@ ${search.solrTrackingSupport.enabled} + + ${search.solrTrackingSupport.ignorePathsForSpecificTypes} + + + + {http://www.alfresco.org/model/content/1.0}person + {http://www.alfresco.org/model/application/1.0}configurations + + diff --git a/config/alfresco/subsystems/Search/lucene/common-search.properties b/config/alfresco/subsystems/Search/lucene/common-search.properties index 5b8746fda2..26eba8023a 100644 --- a/config/alfresco/subsystems/Search/lucene/common-search.properties +++ b/config/alfresco/subsystems/Search/lucene/common-search.properties @@ -1,4 +1,6 @@ search.solrTrackingSupport.enabled=true +search.solrTrackingSupport.ignorePathsForSpecificTypes=false + solr.query.fts.queryConsistency=TRANSACTIONAL_IF_POSSIBLE solr.query.cmis.queryConsistency=TRANSACTIONAL_IF_POSSIBLE solr.query.hybrid.enabled=false diff --git a/config/alfresco/subsystems/Search/noindex/common-search.properties b/config/alfresco/subsystems/Search/noindex/common-search.properties index 52268a1c75..2731e4a51a 100644 --- a/config/alfresco/subsystems/Search/noindex/common-search.properties +++ b/config/alfresco/subsystems/Search/noindex/common-search.properties @@ -1,4 +1,6 @@ search.solrTrackingSupport.enabled=true +search.solrTrackingSupport.ignorePathsForSpecificTypes=false + solr.query.fts.queryConsistency=TRANSACTIONAL_IF_POSSIBLE solr.query.cmis.queryConsistency=TRANSACTIONAL_IF_POSSIBLE solr.query.hybrid.enabled=false diff --git a/config/alfresco/subsystems/Search/solr/common-search.properties b/config/alfresco/subsystems/Search/solr/common-search.properties index 22344e1472..934e3d939d 100644 --- a/config/alfresco/subsystems/Search/solr/common-search.properties +++ b/config/alfresco/subsystems/Search/solr/common-search.properties @@ -1,4 +1,6 @@ search.solrTrackingSupport.enabled=true +search.solrTrackingSupport.ignorePathsForSpecificTypes=false + solr.query.fts.queryConsistency=TRANSACTIONAL_IF_POSSIBLE solr.query.cmis.queryConsistency=TRANSACTIONAL_IF_POSSIBLE solr.query.hybrid.enabled=false \ No newline at end of file diff --git a/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java b/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java index 0c1a9ff588..a18da63ac0 100644 --- a/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java +++ b/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2011 Alfresco Software Limited. + * Copyright (C) 2005-2014 Alfresco Software Limited. * * This file is part of Alfresco * @@ -58,6 +58,8 @@ import org.alfresco.service.cmr.repository.Path; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; import org.alfresco.service.cmr.security.OwnableService; import org.alfresco.service.cmr.security.PermissionService; +import org.alfresco.service.namespace.InvalidQNameException; +import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; import org.alfresco.util.Pair; import org.alfresco.util.PropertyCheck; @@ -78,8 +80,12 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent private OwnableService ownableService; private TenantService tenantService; private DictionaryService dictionaryService; + private NamespaceService namespaceService; private boolean enabled = true; private boolean cacheAncestors =true; + private boolean ignorePathsForSpecificTypes = false; + private Set typesForIgnoringPaths = new HashSet(); + private List typesForIgnoringPathsString; @Override @@ -93,7 +99,22 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent { this.enabled = enabled; } - + + public boolean isIgnorePathsForSpecificTypes() + { + return ignorePathsForSpecificTypes; + } + + public void setIgnorePathsForSpecificTypes(boolean ignorePersonAndConfigurationPaths) + { + this.ignorePathsForSpecificTypes = ignorePersonAndConfigurationPaths; + } + + public void setTypesForIgnoringPaths(List typesForIgnoringPaths) + { + typesForIgnoringPathsString = typesForIgnoringPaths; + } + /** * @param cacheAncestors the cacheAncestors to set */ @@ -136,7 +157,12 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent { this.dictionaryService = dictionaryService; } - + + public void setNamespaceService(NamespaceService namespaceService) + { + this.namespaceService = namespaceService; + } + public void setAclDAO(AclDAO aclDAO) { this.aclDAO = aclDAO; @@ -161,6 +187,29 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent PropertyCheck.mandatory(this, "dictionaryService", dictionaryService); PropertyCheck.mandatory(this, "dictionaryDAO", dictionaryDAO); PropertyCheck.mandatory(this, "aclDAO", aclDAO); + + if ((null != typesForIgnoringPathsString) && (null != namespaceService)) + { + for (String typeQName : typesForIgnoringPathsString) + { + if ((null != typeQName) && !typeQName.isEmpty()) + { + try + { + QName type = QName.resolveToQName(namespaceService, typeQName); + + if (null != dictionaryService.getType(type)) + { + this.typesForIgnoringPaths.add(type); + } + } + catch (InvalidQNameException e) + { + // Just ignore + } + } + } + } } @Override @@ -686,9 +735,8 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent if(includeType) { - QName nodeType = nodeDAO.getNodeType(nodeId); - TypeDefinition type = dictionaryService.getType(nodeType); - if(type != null) + QName nodeType = getNodeType(nodeId); + if(nodeType != null) { nodeMetaData.setNodeType(nodeType); } @@ -725,9 +773,44 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent } } nodeMetaData.setAspects(aspects); - + + boolean ignoreLargeMetadata = ignorePathsForSpecificTypes && shouldTypeBeIgnored(getNodeType(nodeId)); + if (!ignoreLargeMetadata && ignorePathsForSpecificTypes) + { + final List parentIds = new LinkedList(); + nodeDAO.getParentAssocs(nodeId, null, null, true, new ChildAssocRefQueryCallback() + { + @Override + public boolean preLoadNodes() + { + return false; + } + + @Override + public boolean orderResults() + { + return false; + } + + @Override + public boolean handle(Pair childAssocPair, Pair parentNodePair, Pair childNodePair) + { + parentIds.add(parentNodePair.getFirst()); + return false; + } + + @Override + public void done() + { + } + }); + + QName parentType = (!parentIds.isEmpty()) ? (getNodeType(parentIds.iterator().next())) : (null); + ignoreLargeMetadata = shouldTypeBeIgnored(parentType); + } + CategoryPaths categoryPaths = new CategoryPaths(new ArrayList>(), new ArrayList()); - if(includePaths || includeParentAssociations) + if(!ignoreLargeMetadata && (includePaths || includeParentAssociations)) { if(props == null) { @@ -735,26 +818,26 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent } categoryPaths = getCategoryPaths(status.getNodeRef(), aspects, props); } - - if(includePaths) + + if (includePaths && !ignoreLargeMetadata) { - if(props == null) + if (props == null) { props = getProperties(nodeId); } - + List directPaths = nodeDAO.getPaths(new Pair(nodeId, status.getNodeRef()), false); - + Collection> paths = new ArrayList>(directPaths.size() + categoryPaths.getPaths().size()); for (Path path : directPaths) { paths.add(new Pair(path.getBaseNamePath(tenantService), null)); } - for(Pair catPair : categoryPaths.getPaths()) + for (Pair catPair : categoryPaths.getPaths()) { - paths.add(new Pair(catPair.getFirst().getBaseNamePath(tenantService), catPair.getSecond())); + paths.add(new Pair(catPair.getFirst().getBaseNamePath(tenantService), catPair.getSecond())); } - + nodeMetaData.setPaths(paths); } @@ -781,7 +864,16 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent public boolean handle(Pair childAssocPair, Pair parentNodePair, Pair childNodePair) { - childAssocs.add(tenantService.getBaseName(childAssocPair.getSecond(), true)); + boolean addCurrentChildAssoc = true; + if (ignorePathsForSpecificTypes) + { + QName nodeType = nodeDAO.getNodeType(childNodePair.getFirst()); + addCurrentChildAssoc = !shouldTypeBeIgnored(nodeType); + } + if (addCurrentChildAssoc) + { + childAssocs.add(tenantService.getBaseName(childAssocPair.getSecond(), true)); + } return true; } @@ -814,7 +906,16 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent public boolean handle(Pair childAssocPair, Pair parentNodePair, Pair childNodePair) { - childIds.add(childNodePair.getFirst()); + boolean addCurrentId = true; + if (ignorePathsForSpecificTypes) + { + QName nodeType = nodeDAO.getNodeType(childNodePair.getFirst()); + addCurrentId = !shouldTypeBeIgnored(nodeType); + } + if (addCurrentId) + { + childIds.add(childNodePair.getFirst()); + } return true; } @@ -825,8 +926,8 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent }); nodeMetaData.setChildIds(childIds); } - - if(includeParentAssociations) + + if(includeParentAssociations && !ignoreLargeMetadata) { final List parentAssocs = new ArrayList(100); nodeDAO.getParentAssocs(nodeId, null, null, null, new ChildAssocRefQueryCallback() @@ -891,7 +992,35 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent rowHandler.processResult(nodeMetaData); } } - + + private QName getNodeType(Long nodeId) + { + QName result = nodeDAO.getNodeType(nodeId); + TypeDefinition type = dictionaryService.getType(result); + return (null == type) ? (null) : (result); + } + + private boolean shouldTypeBeIgnored(QName nodeType) + { + if (null != nodeType) + { + if (typesForIgnoringPaths.contains(nodeType)) + { + return true; + } + + for (QName type : typesForIgnoringPaths) + { + if (dictionaryService.isSubClass(nodeType, type)) + { + return true; + } + } + } + + return false; + } + /** * {@inheritDoc} */