diff --git a/config/alfresco/ibatis/org.hibernate.dialect.Dialect/solr-common-SqlMap.xml b/config/alfresco/ibatis/org.hibernate.dialect.Dialect/solr-common-SqlMap.xml index 1d08379bf4..50fcb92fcb 100644 --- a/config/alfresco/ibatis/org.hibernate.dialect.Dialect/solr-common-SqlMap.xml +++ b/config/alfresco/ibatis/org.hibernate.dialect.Dialect/solr-common-SqlMap.xml @@ -30,6 +30,7 @@ + @@ -119,12 +120,29 @@ node.uuid as uuid, node.acl_id as acl_id, store.protocol as protocol, - store.identifier as identifier + store.identifier as identifier, + + + null as shard_key + + + node.audit_created as shard_key + + + node.audit_modified as shard_key + + + shardkey.string_value as shard_key + + from alf_transaction txn join alf_node node on (txn.id = node.transaction_id) join alf_store store on (store.id = node.store_id) left outer join alf_node_properties np on (np.node_id = node.id and np.qname_id = #{originalIdPropQNameId}) + + left outer join alf_node_properties shardkey on (shardkey.node_id = node.id and shardkey.qname_id = #{shardPropertyQNameId}) + diff --git a/config/alfresco/subsystems/Search/common-search-context.xml b/config/alfresco/subsystems/Search/common-search-context.xml index c2b1d5c974..b433b88c7c 100644 --- a/config/alfresco/subsystems/Search/common-search-context.xml +++ b/config/alfresco/subsystems/Search/common-search-context.xml @@ -20,6 +20,7 @@ + diff --git a/source/java/org/alfresco/repo/domain/node/Node.java b/source/java/org/alfresco/repo/domain/node/Node.java index 019b044ab2..b034651ff7 100644 --- a/source/java/org/alfresco/repo/domain/node/Node.java +++ b/source/java/org/alfresco/repo/domain/node/Node.java @@ -72,6 +72,8 @@ public interface Node extends NodeIdAndAclId public abstract TransactionEntity getTransaction(); - public abstract AuditablePropertiesEntity getAuditableProperties(); + public abstract AuditablePropertiesEntity getAuditableProperties(); + + public abstract String getShardKey(); } \ No newline at end of file diff --git a/source/java/org/alfresco/repo/domain/node/NodeEntity.java b/source/java/org/alfresco/repo/domain/node/NodeEntity.java index 55e95ec602..eb12b0c25d 100644 --- a/source/java/org/alfresco/repo/domain/node/NodeEntity.java +++ b/source/java/org/alfresco/repo/domain/node/NodeEntity.java @@ -54,7 +54,8 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo private Long localeId; private Long aclId; private TransactionEntity transaction; - private AuditablePropertiesEntity auditableProperties; + private AuditablePropertiesEntity auditableProperties; + private String shardKey; /** * Required default constructor @@ -89,7 +90,8 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo this.localeId = node.getLocaleId(); this.aclId = node.getAclId(); this.transaction = node.getTransaction(); - this.auditableProperties = node.getAuditableProperties(); + this.auditableProperties = node.getAuditableProperties(); + this.shardKey = node.getShardKey(); } @Override @@ -137,7 +139,8 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo .append(", localeId=").append(localeId) .append(", aclId=").append(aclId) .append(", transaction=").append(transaction) - .append(", auditProps=").append(auditableProperties) + .append(", auditProps=").append(auditableProperties) + .append(", shardKey=").append(shardKey) .append("]"); return sb.toString(); } @@ -331,5 +334,18 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo { checkLock(); this.auditableProperties = auditableProperties; - } + } + + @Override + public String getShardKey() + { + return this.shardKey; + } + + public synchronized void setShardKey(String shardKey) + { + this.shardKey = shardKey; + } + + } diff --git a/source/java/org/alfresco/repo/domain/solr/NodeParametersEntity.java b/source/java/org/alfresco/repo/domain/solr/NodeParametersEntity.java index 87b7a7d441..d3da1fef84 100644 --- a/source/java/org/alfresco/repo/domain/solr/NodeParametersEntity.java +++ b/source/java/org/alfresco/repo/domain/solr/NodeParametersEntity.java @@ -48,7 +48,8 @@ public class NodeParametersEntity extends NodeParameters private List includeAspectIds; private List excludeAspectIds; - private Long originalIdPropQNameId; + private Long originalIdPropQNameId; + private Long shardPropertyQNameId; /** * Public constructor, but not generally useful @@ -160,5 +161,17 @@ public class NodeParametersEntity extends NodeParameters public boolean isIncludeNodesTable() { return (getFromNodeId() != null || getToNodeId() != null || getIncludeTypeIds() != null || getExcludeTypeIds() != null || getIncludeAspectIds() != null || getExcludeAspectIds() != null); - } + } + + public Long getShardPropertyQNameId() + { + return this.shardPropertyQNameId; + } + + public void setShardPropertyQNameId(Long shardPropertyQNameId) + { + this.shardPropertyQNameId = shardPropertyQNameId; + } + + } diff --git a/source/java/org/alfresco/repo/domain/solr/SOLRDAO.java b/source/java/org/alfresco/repo/domain/solr/SOLRDAO.java index 729b98dd89..e25b311b20 100644 --- a/source/java/org/alfresco/repo/domain/solr/SOLRDAO.java +++ b/source/java/org/alfresco/repo/domain/solr/SOLRDAO.java @@ -31,7 +31,8 @@ import org.alfresco.repo.domain.node.Node; import org.alfresco.repo.solr.Acl; import org.alfresco.repo.solr.AclChangeSet; import org.alfresco.repo.solr.NodeParameters; -import org.alfresco.repo.solr.Transaction; +import org.alfresco.repo.solr.Transaction; +import org.alfresco.service.namespace.QName; /** * DAO support for SOLR web scripts. @@ -77,8 +78,9 @@ public interface SOLRDAO /** * Get the nodes satisfying the constraints in nodeParameters * - * @param nodeParameters set of constraints for which nodes to return + * @param nodeParameters set of constraints for which nodes to return + * @param shardPropertQName * @return list of matching nodes */ - public List getNodes(NodeParameters nodeParameters); + public List getNodes(NodeParameters nodeParameters, QName shardPropertQName); } diff --git a/source/java/org/alfresco/repo/domain/solr/ibatis/SOLRDAOImpl.java b/source/java/org/alfresco/repo/domain/solr/ibatis/SOLRDAOImpl.java index ffc8fb7d42..6b0fdbf949 100644 --- a/source/java/org/alfresco/repo/domain/solr/ibatis/SOLRDAOImpl.java +++ b/source/java/org/alfresco/repo/domain/solr/ibatis/SOLRDAOImpl.java @@ -36,11 +36,13 @@ import org.alfresco.repo.domain.qname.QNameDAO; import org.alfresco.repo.domain.solr.AclEntity; import org.alfresco.repo.domain.solr.NodeParametersEntity; import org.alfresco.repo.domain.solr.SOLRDAO; -import org.alfresco.repo.domain.solr.SOLRTrackingParameters; +import org.alfresco.repo.domain.solr.SOLRTrackingParameters; +import org.alfresco.repo.search.impl.QueryParserUtils; import org.alfresco.repo.solr.Acl; import org.alfresco.repo.solr.AclChangeSet; import org.alfresco.repo.solr.NodeParameters; -import org.alfresco.repo.solr.Transaction; +import org.alfresco.repo.solr.Transaction; +import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; import org.alfresco.util.Pair; import org.alfresco.util.PropertyCheck; @@ -95,8 +97,8 @@ public class SOLRDAOImpl implements SOLRDAO // We simulate an ID for the sys:deleted type Pair deletedTypeQNamePair = qnameDAO.getQName(ContentModel.TYPE_DELETED); - Long deletedTypeQNameId = deletedTypeQNamePair == null ? -1L : deletedTypeQNamePair.getFirst(); - + Long deletedTypeQNameId = deletedTypeQNamePair == null ? -1L : deletedTypeQNamePair.getFirst(); + SOLRTrackingParameters params = new SOLRTrackingParameters(deletedTypeQNameId); params.setFromIdInclusive(minAclChangeSetId); params.setFromCommitTimeInclusive(fromCommitTime); @@ -197,9 +199,29 @@ public class SOLRDAOImpl implements SOLRDAO * {@inheritDoc} */ @SuppressWarnings("unchecked") - public List getNodes(NodeParameters nodeParameters) - { - NodeParametersEntity params = new NodeParametersEntity(nodeParameters, qnameDAO); + public List getNodes(NodeParameters nodeParameters, QName shardPropertyQName) + { + NodeParametersEntity params = new NodeParametersEntity(nodeParameters, qnameDAO); + + if(shardPropertyQName !=null) + { + if(shardPropertyQName.equals(ContentModel.PROP_CREATED)) + { + params.setShardPropertyQNameId(-1L); + } + else if (shardPropertyQName.equals(ContentModel.PROP_MODIFIED)) + { + params.setShardPropertyQNameId(-2L); + } + else + { + Pair propertyQNamePair = qnameDAO.getQName(shardPropertyQName); + if(propertyQNamePair != null) + { + params.setShardPropertyQNameId(propertyQNamePair.getFirst()); + } + } + } if(nodeParameters.getMaxResults() != 0 && nodeParameters.getMaxResults() != Integer.MAX_VALUE) { diff --git a/source/java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguage.java b/source/java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguage.java index e78316aba4..f6e2672b06 100644 --- a/source/java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguage.java +++ b/source/java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguage.java @@ -331,7 +331,7 @@ public class DbOrIndexSwitchingQueryLanguage extends AbstractLuceneQueryLanguage // TODO: setToTxnId(null) when SolrDAO behaviour is fixed. nodeParameters.setToTxnId(Long.MAX_VALUE); stopWatch.start("get changed nodes"); - List changedNodeList = solrDao.getNodes(nodeParameters); + List changedNodeList = solrDao.getNodes(nodeParameters, null); stopWatch.stop(); if (logger.isDebugEnabled()) { diff --git a/source/java/org/alfresco/repo/solr/NodeParameters.java b/source/java/org/alfresco/repo/solr/NodeParameters.java index 179dca602e..e01c0b753d 100644 --- a/source/java/org/alfresco/repo/solr/NodeParameters.java +++ b/source/java/org/alfresco/repo/solr/NodeParameters.java @@ -54,7 +54,9 @@ public class NodeParameters private Set excludeNodeTypes; private Set includeAspects; - private Set excludeAspects; + private Set excludeAspects; + + private String shardProperty; public int getMaxResults() { @@ -179,5 +181,15 @@ public class NodeParameters public void setExcludeAspects(Set excludeAspects) { this.excludeAspects = excludeAspects; + } + + public String getShardProperty() + { + return this.shardProperty; + } + + public void setShardProperty(String shardProperty) + { + this.shardProperty = shardProperty; } } diff --git a/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java b/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java index 07f093f968..53e53b0d4f 100644 --- a/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java +++ b/source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java @@ -52,7 +52,8 @@ import org.alfresco.repo.domain.solr.SOLRDAO; import org.alfresco.repo.index.shard.ShardRegistry; import org.alfresco.repo.index.shard.ShardState; import org.alfresco.repo.search.AspectIndexFilter; -import org.alfresco.repo.search.TypeIndexFilter; +import org.alfresco.repo.search.TypeIndexFilter; +import org.alfresco.repo.search.impl.QueryParserUtils; import org.alfresco.repo.security.authentication.AuthenticationUtil; import org.alfresco.repo.tenant.TenantService; import org.alfresco.service.cmr.dictionary.AspectDefinition; @@ -69,10 +70,13 @@ import org.alfresco.service.cmr.repository.Path; import org.alfresco.service.cmr.repository.Path.ChildAssocElement; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; import org.alfresco.service.cmr.security.OwnableService; -import org.alfresco.service.cmr.security.PermissionService; +import org.alfresco.service.cmr.security.PermissionService; +import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; import org.alfresco.util.Pair; -import org.alfresco.util.PropertyCheck; +import org.alfresco.util.PropertyCheck; + +import com.sun.xml.txw2.NamespaceResolver; /** * Component providing data for SOLR tracking @@ -94,7 +98,8 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent private boolean cacheAncestors =true; private TypeIndexFilter typeIndexFilter; private AspectIndexFilter aspectIndexFilter; - private ShardRegistry shardRegistry; + private ShardRegistry shardRegistry; + private NamespaceService namespaceService; @Override @@ -177,6 +182,11 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent this.shardRegistry = shardRegistry; } + public void setNamespaceService(NamespaceService namespaceService) + { + this.namespaceService = namespaceService; + } + /** * Initialize */ @@ -192,7 +202,8 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent PropertyCheck.mandatory(this, "dictionaryDAO", dictionaryDAO); PropertyCheck.mandatory(this, "aclDAO", aclDAO); PropertyCheck.mandatory(this, "typeIndexFilter", typeIndexFilter); - PropertyCheck.mandatory(this, "aspectIndexFilter", aspectIndexFilter); + PropertyCheck.mandatory(this, "aspectIndexFilter", aspectIndexFilter); + PropertyCheck.mandatory(this, "namespaceService", namespaceService); } @Override @@ -343,8 +354,23 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent public void getNodes(NodeParameters nodeParameters, NodeQueryCallback callback) { if(enabled) - { - List nodes = solrDAO.getNodes(nodeParameters); + { + QName shardPropertQName = null; + if(nodeParameters.getShardProperty() != null) + { + PropertyDefinition pdef = QueryParserUtils.matchPropertyDefinition(NamespaceService.CONTENT_MODEL_1_0_URI, namespaceService, dictionaryService, nodeParameters.getShardProperty()); + if(pdef == null) + { + throw new AlfrescoRuntimeException("Invalid shard property: "+nodeParameters.getShardProperty()); + } + if((!pdef.getDataType().getName().equals(DataTypeDefinition.TEXT)) && (!pdef.getDataType().getName().equals(DataTypeDefinition.DATE)) && (!pdef.getDataType().getName().equals(DataTypeDefinition.DATETIME))) + { + throw new AlfrescoRuntimeException("Unsupported shard property type: "+(pdef.getDataType().getName() + " for " +nodeParameters.getShardProperty())); + } + shardPropertQName = pdef.getName(); + } + + List nodes = solrDAO.getNodes(nodeParameters, shardPropertQName); for (Node node : nodes) { diff --git a/source/test-java/org/alfresco/repo/domain/solr/SOLRDAOTest.java b/source/test-java/org/alfresco/repo/domain/solr/SOLRDAOTest.java index 58a83e075c..29b9144e44 100644 --- a/source/test-java/org/alfresco/repo/domain/solr/SOLRDAOTest.java +++ b/source/test-java/org/alfresco/repo/domain/solr/SOLRDAOTest.java @@ -102,7 +102,7 @@ public class SOLRDAOTest extends TestCase @Override public List execute() throws Throwable { - return solrDAO.getNodes(nodeParameters); + return solrDAO.getNodes(nodeParameters, null); } }, true); } diff --git a/source/test-java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguageTest.java b/source/test-java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguageTest.java index 0dc9cb0af5..355fca747f 100644 --- a/source/test-java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguageTest.java +++ b/source/test-java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguageTest.java @@ -88,7 +88,7 @@ public class DbOrIndexSwitchingQueryLanguageTest when(indexQueryLang.executeQuery(argThat(isSearchParamsSinceTxId(null)), eq(admLuceneSearcher))).thenReturn(indexResults); when(indexResults.getLastIndexedTxId()).thenReturn(80L); when(dbQueryLang.executeQuery(argThat(isSearchParamsSinceTxId(80L)), eq(admLuceneSearcher))).thenReturn(dbResults); - when(solrDAO.getNodes(argThat(isNodeParamsFromTxnId(81L)))).thenReturn(changedNodes); + when(solrDAO.getNodes(argThat(isNodeParamsFromTxnId(81L)), null)).thenReturn(changedNodes); searchParameters.setQueryConsistency(QueryConsistency.HYBRID); diff --git a/source/test-java/org/alfresco/repo/search/impl/solr/TestNode.java b/source/test-java/org/alfresco/repo/search/impl/solr/TestNode.java index 23126a340e..5d760ae5fd 100644 --- a/source/test-java/org/alfresco/repo/search/impl/solr/TestNode.java +++ b/source/test-java/org/alfresco/repo/search/impl/solr/TestNode.java @@ -131,6 +131,12 @@ class TestNode implements Node public AuditablePropertiesEntity getAuditableProperties() { return null; + } + + @Override + public String getShardKey() + { + return null; } }