mirror of
https://github.com/Alfresco/alfresco-community-repo.git
synced 2025-10-15 15:02:20 +00:00
Merged searchapi (5.2.1) to 5.2.N (5.2.1)
130022 gjames: Merged 5.2.N-AHIND (5.2.1) to searchapi (5.2.1) 129926 ahind: SEARCH-31 SOLR 6 - Shard based on date - SOLR implementation - Support returning a date, datetime or string property to use for sharding git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/BRANCHES/DEV/5.2.N/root@130285 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
@@ -30,6 +30,7 @@
|
||||
<result property="store.protocol" column="protocol" jdbcType="VARCHAR" javaType="java.lang.String"/>
|
||||
<result property="store.identifier" column="identifier" jdbcType="VARCHAR" javaType="java.lang.String"/>
|
||||
<result property="uuid" column="uuid" jdbcType="VARCHAR" javaType="java.lang.String"/>
|
||||
<result property="shardKey" column="shard_key" jdbcType="VARCHAR" javaType="java.lang.String"/>
|
||||
</resultMap>
|
||||
|
||||
<!-- SELECTS -->
|
||||
@@ -119,12 +120,29 @@
|
||||
node.uuid as uuid,
|
||||
node.acl_id as acl_id,
|
||||
store.protocol as protocol,
|
||||
store.identifier as identifier
|
||||
store.identifier as identifier,
|
||||
<choose>
|
||||
<when test="shardPropertyQNameId == null">
|
||||
null as shard_key
|
||||
</when>
|
||||
<when test="shardPropertyQNameId == -1">
|
||||
node.audit_created as shard_key
|
||||
</when>
|
||||
<when test="shardPropertyQNameId == -2">
|
||||
node.audit_modified as shard_key
|
||||
</when>
|
||||
<otherwise>
|
||||
shardkey.string_value as shard_key
|
||||
</otherwise>
|
||||
</choose>
|
||||
from
|
||||
alf_transaction txn
|
||||
join alf_node node on (txn.id = node.transaction_id)
|
||||
join alf_store store on (store.id = node.store_id)
|
||||
left outer join alf_node_properties np on (np.node_id = node.id and np.qname_id = #{originalIdPropQNameId})
|
||||
<if test="shardPropertyQNameId != null and shardPropertyQNameId >= 0" >
|
||||
left outer join alf_node_properties shardkey on (shardkey.node_id = node.id and shardkey.qname_id = #{shardPropertyQNameId})
|
||||
</if>
|
||||
<where>
|
||||
<choose>
|
||||
<when test="transactionIds != null">
|
||||
|
@@ -20,6 +20,7 @@
|
||||
</property>
|
||||
<property name="typeIndexFilter" ref="search.TypeIndexFilter" />
|
||||
<property name="aspectIndexFilter" ref="search.AspectIndexFilter" />
|
||||
<property name="namespaceService" ref="namespaceService" />
|
||||
</bean>
|
||||
|
||||
<!-- Ignore indexing by node type or by node aspects -->
|
||||
|
@@ -74,4 +74,6 @@ public interface Node extends NodeIdAndAclId
|
||||
|
||||
public abstract AuditablePropertiesEntity getAuditableProperties();
|
||||
|
||||
public abstract String getShardKey();
|
||||
|
||||
}
|
@@ -55,6 +55,7 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
|
||||
private Long aclId;
|
||||
private TransactionEntity transaction;
|
||||
private AuditablePropertiesEntity auditableProperties;
|
||||
private String shardKey;
|
||||
|
||||
/**
|
||||
* Required default constructor
|
||||
@@ -90,6 +91,7 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
|
||||
this.aclId = node.getAclId();
|
||||
this.transaction = node.getTransaction();
|
||||
this.auditableProperties = node.getAuditableProperties();
|
||||
this.shardKey = node.getShardKey();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -138,6 +140,7 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
|
||||
.append(", aclId=").append(aclId)
|
||||
.append(", transaction=").append(transaction)
|
||||
.append(", auditProps=").append(auditableProperties)
|
||||
.append(", shardKey=").append(shardKey)
|
||||
.append("]");
|
||||
return sb.toString();
|
||||
}
|
||||
@@ -332,4 +335,17 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
|
||||
checkLock();
|
||||
this.auditableProperties = auditableProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getShardKey()
|
||||
{
|
||||
return this.shardKey;
|
||||
}
|
||||
|
||||
public synchronized void setShardKey(String shardKey)
|
||||
{
|
||||
this.shardKey = shardKey;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
@@ -49,6 +49,7 @@ public class NodeParametersEntity extends NodeParameters
|
||||
private List<Long> excludeAspectIds;
|
||||
|
||||
private Long originalIdPropQNameId;
|
||||
private Long shardPropertyQNameId;
|
||||
|
||||
/**
|
||||
* Public constructor, but not generally useful
|
||||
@@ -161,4 +162,16 @@ public class NodeParametersEntity extends NodeParameters
|
||||
{
|
||||
return (getFromNodeId() != null || getToNodeId() != null || getIncludeTypeIds() != null || getExcludeTypeIds() != null || getIncludeAspectIds() != null || getExcludeAspectIds() != null);
|
||||
}
|
||||
|
||||
public Long getShardPropertyQNameId()
|
||||
{
|
||||
return this.shardPropertyQNameId;
|
||||
}
|
||||
|
||||
public void setShardPropertyQNameId(Long shardPropertyQNameId)
|
||||
{
|
||||
this.shardPropertyQNameId = shardPropertyQNameId;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
@@ -32,6 +32,7 @@ import org.alfresco.repo.solr.Acl;
|
||||
import org.alfresco.repo.solr.AclChangeSet;
|
||||
import org.alfresco.repo.solr.NodeParameters;
|
||||
import org.alfresco.repo.solr.Transaction;
|
||||
import org.alfresco.service.namespace.QName;
|
||||
|
||||
/**
|
||||
* DAO support for SOLR web scripts.
|
||||
@@ -78,7 +79,8 @@ public interface SOLRDAO
|
||||
* Get the nodes satisfying the constraints in nodeParameters
|
||||
*
|
||||
* @param nodeParameters set of constraints for which nodes to return
|
||||
* @param shardPropertQName
|
||||
* @return list of matching nodes
|
||||
*/
|
||||
public List<Node> getNodes(NodeParameters nodeParameters);
|
||||
public List<Node> getNodes(NodeParameters nodeParameters, QName shardPropertQName);
|
||||
}
|
||||
|
@@ -37,10 +37,12 @@ import org.alfresco.repo.domain.solr.AclEntity;
|
||||
import org.alfresco.repo.domain.solr.NodeParametersEntity;
|
||||
import org.alfresco.repo.domain.solr.SOLRDAO;
|
||||
import org.alfresco.repo.domain.solr.SOLRTrackingParameters;
|
||||
import org.alfresco.repo.search.impl.QueryParserUtils;
|
||||
import org.alfresco.repo.solr.Acl;
|
||||
import org.alfresco.repo.solr.AclChangeSet;
|
||||
import org.alfresco.repo.solr.NodeParameters;
|
||||
import org.alfresco.repo.solr.Transaction;
|
||||
import org.alfresco.service.namespace.NamespaceService;
|
||||
import org.alfresco.service.namespace.QName;
|
||||
import org.alfresco.util.Pair;
|
||||
import org.alfresco.util.PropertyCheck;
|
||||
@@ -197,10 +199,30 @@ public class SOLRDAOImpl implements SOLRDAO
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public List<Node> getNodes(NodeParameters nodeParameters)
|
||||
public List<Node> getNodes(NodeParameters nodeParameters, QName shardPropertyQName)
|
||||
{
|
||||
NodeParametersEntity params = new NodeParametersEntity(nodeParameters, qnameDAO);
|
||||
|
||||
if(shardPropertyQName !=null)
|
||||
{
|
||||
if(shardPropertyQName.equals(ContentModel.PROP_CREATED))
|
||||
{
|
||||
params.setShardPropertyQNameId(-1L);
|
||||
}
|
||||
else if (shardPropertyQName.equals(ContentModel.PROP_MODIFIED))
|
||||
{
|
||||
params.setShardPropertyQNameId(-2L);
|
||||
}
|
||||
else
|
||||
{
|
||||
Pair<Long, QName> propertyQNamePair = qnameDAO.getQName(shardPropertyQName);
|
||||
if(propertyQNamePair != null)
|
||||
{
|
||||
params.setShardPropertyQNameId(propertyQNamePair.getFirst());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(nodeParameters.getMaxResults() != 0 && nodeParameters.getMaxResults() != Integer.MAX_VALUE)
|
||||
{
|
||||
return template.selectList(
|
||||
|
@@ -331,7 +331,7 @@ public class DbOrIndexSwitchingQueryLanguage extends AbstractLuceneQueryLanguage
|
||||
// TODO: setToTxnId(null) when SolrDAO behaviour is fixed.
|
||||
nodeParameters.setToTxnId(Long.MAX_VALUE);
|
||||
stopWatch.start("get changed nodes");
|
||||
List<Node> changedNodeList = solrDao.getNodes(nodeParameters);
|
||||
List<Node> changedNodeList = solrDao.getNodes(nodeParameters, null);
|
||||
stopWatch.stop();
|
||||
if (logger.isDebugEnabled())
|
||||
{
|
||||
|
@@ -56,6 +56,8 @@ public class NodeParameters
|
||||
private Set<QName> includeAspects;
|
||||
private Set<QName> excludeAspects;
|
||||
|
||||
private String shardProperty;
|
||||
|
||||
public int getMaxResults()
|
||||
{
|
||||
return maxResults;
|
||||
@@ -180,4 +182,14 @@ public class NodeParameters
|
||||
{
|
||||
this.excludeAspects = excludeAspects;
|
||||
}
|
||||
|
||||
public String getShardProperty()
|
||||
{
|
||||
return this.shardProperty;
|
||||
}
|
||||
|
||||
public void setShardProperty(String shardProperty)
|
||||
{
|
||||
this.shardProperty = shardProperty;
|
||||
}
|
||||
}
|
||||
|
@@ -53,6 +53,7 @@ import org.alfresco.repo.index.shard.ShardRegistry;
|
||||
import org.alfresco.repo.index.shard.ShardState;
|
||||
import org.alfresco.repo.search.AspectIndexFilter;
|
||||
import org.alfresco.repo.search.TypeIndexFilter;
|
||||
import org.alfresco.repo.search.impl.QueryParserUtils;
|
||||
import org.alfresco.repo.security.authentication.AuthenticationUtil;
|
||||
import org.alfresco.repo.tenant.TenantService;
|
||||
import org.alfresco.service.cmr.dictionary.AspectDefinition;
|
||||
@@ -70,10 +71,13 @@ import org.alfresco.service.cmr.repository.Path.ChildAssocElement;
|
||||
import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
|
||||
import org.alfresco.service.cmr.security.OwnableService;
|
||||
import org.alfresco.service.cmr.security.PermissionService;
|
||||
import org.alfresco.service.namespace.NamespaceService;
|
||||
import org.alfresco.service.namespace.QName;
|
||||
import org.alfresco.util.Pair;
|
||||
import org.alfresco.util.PropertyCheck;
|
||||
|
||||
import com.sun.xml.txw2.NamespaceResolver;
|
||||
|
||||
/**
|
||||
* Component providing data for SOLR tracking
|
||||
*
|
||||
@@ -95,6 +99,7 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
|
||||
private TypeIndexFilter typeIndexFilter;
|
||||
private AspectIndexFilter aspectIndexFilter;
|
||||
private ShardRegistry shardRegistry;
|
||||
private NamespaceService namespaceService;
|
||||
|
||||
|
||||
@Override
|
||||
@@ -177,6 +182,11 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
|
||||
this.shardRegistry = shardRegistry;
|
||||
}
|
||||
|
||||
public void setNamespaceService(NamespaceService namespaceService)
|
||||
{
|
||||
this.namespaceService = namespaceService;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize
|
||||
*/
|
||||
@@ -193,6 +203,7 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
|
||||
PropertyCheck.mandatory(this, "aclDAO", aclDAO);
|
||||
PropertyCheck.mandatory(this, "typeIndexFilter", typeIndexFilter);
|
||||
PropertyCheck.mandatory(this, "aspectIndexFilter", aspectIndexFilter);
|
||||
PropertyCheck.mandatory(this, "namespaceService", namespaceService);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -344,7 +355,22 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
|
||||
{
|
||||
if(enabled)
|
||||
{
|
||||
List<Node> nodes = solrDAO.getNodes(nodeParameters);
|
||||
QName shardPropertQName = null;
|
||||
if(nodeParameters.getShardProperty() != null)
|
||||
{
|
||||
PropertyDefinition pdef = QueryParserUtils.matchPropertyDefinition(NamespaceService.CONTENT_MODEL_1_0_URI, namespaceService, dictionaryService, nodeParameters.getShardProperty());
|
||||
if(pdef == null)
|
||||
{
|
||||
throw new AlfrescoRuntimeException("Invalid shard property: "+nodeParameters.getShardProperty());
|
||||
}
|
||||
if((!pdef.getDataType().getName().equals(DataTypeDefinition.TEXT)) && (!pdef.getDataType().getName().equals(DataTypeDefinition.DATE)) && (!pdef.getDataType().getName().equals(DataTypeDefinition.DATETIME)))
|
||||
{
|
||||
throw new AlfrescoRuntimeException("Unsupported shard property type: "+(pdef.getDataType().getName() + " for " +nodeParameters.getShardProperty()));
|
||||
}
|
||||
shardPropertQName = pdef.getName();
|
||||
}
|
||||
|
||||
List<Node> nodes = solrDAO.getNodes(nodeParameters, shardPropertQName);
|
||||
|
||||
for (Node node : nodes)
|
||||
{
|
||||
|
@@ -102,7 +102,7 @@ public class SOLRDAOTest extends TestCase
|
||||
@Override
|
||||
public List<Node> execute() throws Throwable
|
||||
{
|
||||
return solrDAO.getNodes(nodeParameters);
|
||||
return solrDAO.getNodes(nodeParameters, null);
|
||||
}
|
||||
}, true);
|
||||
}
|
||||
|
@@ -88,7 +88,7 @@ public class DbOrIndexSwitchingQueryLanguageTest
|
||||
when(indexQueryLang.executeQuery(argThat(isSearchParamsSinceTxId(null)), eq(admLuceneSearcher))).thenReturn(indexResults);
|
||||
when(indexResults.getLastIndexedTxId()).thenReturn(80L);
|
||||
when(dbQueryLang.executeQuery(argThat(isSearchParamsSinceTxId(80L)), eq(admLuceneSearcher))).thenReturn(dbResults);
|
||||
when(solrDAO.getNodes(argThat(isNodeParamsFromTxnId(81L)))).thenReturn(changedNodes);
|
||||
when(solrDAO.getNodes(argThat(isNodeParamsFromTxnId(81L)), null)).thenReturn(changedNodes);
|
||||
|
||||
searchParameters.setQueryConsistency(QueryConsistency.HYBRID);
|
||||
|
||||
|
@@ -133,4 +133,10 @@ class TestNode implements Node
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getShardKey()
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
Reference in New Issue
Block a user