inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-10-15 15:02:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merged searchapi (5.2.1) to 5.2.N (5.2.1)

Browse Source 
130022 gjames: Merged 5.2.N-AHIND (5.2.1) to searchapi (5.2.1)
      129926 ahind: SEARCH-31    SOLR 6 - Shard based on date - SOLR implementation
      - Support returning a date, datetime or string property to use for sharding


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/BRANCHES/DEV/5.2.N/root@130285 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Gethin James
2016-09-06 15:35:59 +00:00
parent 905c97c8f8
commit 1672f28cb1
13 changed files with 147 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
config/alfresco/ibatis/org.hibernate.dialect.Dialect/solr-common-SqlMap.xml
View File

@@ -30,6 +30,7 @@
<result property="store.protocol" column="protocol" jdbcType="VARCHAR" javaType="java.lang.String"/>
<result property="store.identifier" column="identifier" jdbcType="VARCHAR" javaType="java.lang.String"/>
<result property="uuid" column="uuid" jdbcType="VARCHAR" javaType="java.lang.String"/>
<result property="shardKey" column="shard_key" jdbcType="VARCHAR" javaType="java.lang.String"/>
</resultMap>
<!-- SELECTS -->
@@ -119,12 +120,29 @@
node.uuid as uuid,
node.acl_id as acl_id,
store.protocol as protocol,
store.identifier as identifier
store.identifier as identifier,
<choose>
<when test="shardPropertyQNameId == null">
null as shard_key
</when>
<when test="shardPropertyQNameId == -1">
node.audit_created as shard_key
</when>
<when test="shardPropertyQNameId == -2">
node.audit_modified as shard_key
</when>
<otherwise>
shardkey.string_value as shard_key
</otherwise>
</choose>
from
alf_transaction txn
join alf_node node on (txn.id = node.transaction_id)
join alf_store store on (store.id = node.store_id)
left outer join alf_node_properties np on (np.node_id = node.id and np.qname_id = #{originalIdPropQNameId})
<if test="shardPropertyQNameId != null and shardPropertyQNameId >= 0" >
left outer join alf_node_properties shardkey on (shardkey.node_id = node.id and shardkey.qname_id = #{shardPropertyQNameId})
</if>
<where>
<choose>
<when test="transactionIds != null">

1
config/alfresco/subsystems/Search/common-search-context.xml
View File

@@ -20,6 +20,7 @@
</property>
<property name="typeIndexFilter" ref="search.TypeIndexFilter" />
<property name="aspectIndexFilter" ref="search.AspectIndexFilter" />
<property name="namespaceService" ref="namespaceService" />
</bean>
<!-- Ignore indexing by node type or by node aspects -->

2
source/java/org/alfresco/repo/domain/node/Node.java
View File

@@ -74,4 +74,6 @@ public interface Node extends NodeIdAndAclId
public abstract AuditablePropertiesEntity getAuditableProperties();
public abstract String getShardKey();
}

16
source/java/org/alfresco/repo/domain/node/NodeEntity.java
View File

@@ -55,6 +55,7 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
private Long aclId;
private TransactionEntity transaction;
private AuditablePropertiesEntity auditableProperties;
private String shardKey;
/**
* Required default constructor
@@ -90,6 +91,7 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
this.aclId = node.getAclId();
this.transaction = node.getTransaction();
this.auditableProperties = node.getAuditableProperties();
this.shardKey = node.getShardKey();
}
@Override
@@ -138,6 +140,7 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
.append(", aclId=").append(aclId)
.append(", transaction=").append(transaction)
.append(", auditProps=").append(auditableProperties)
.append(", shardKey=").append(shardKey)
.append("]");
return sb.toString();
}
@@ -332,4 +335,17 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
checkLock();
this.auditableProperties = auditableProperties;
}
@Override
public String getShardKey()
{
return this.shardKey;
}
public synchronized void setShardKey(String shardKey)
{
this.shardKey = shardKey;
}
}

13
source/java/org/alfresco/repo/domain/solr/NodeParametersEntity.java
View File

@@ -49,6 +49,7 @@ public class NodeParametersEntity extends NodeParameters
private List<Long> excludeAspectIds;
private Long originalIdPropQNameId;
private Long shardPropertyQNameId;
/**
* Public constructor, but not generally useful
@@ -161,4 +162,16 @@ public class NodeParametersEntity extends NodeParameters
{
return (getFromNodeId() != null || getToNodeId() != null || getIncludeTypeIds() != null || getExcludeTypeIds() != null || getIncludeAspectIds() != null || getExcludeAspectIds() != null);
}
public Long getShardPropertyQNameId()
{
return this.shardPropertyQNameId;
}
public void setShardPropertyQNameId(Long shardPropertyQNameId)
{
this.shardPropertyQNameId = shardPropertyQNameId;
}
}

4
source/java/org/alfresco/repo/domain/solr/SOLRDAO.java
View File

@@ -32,6 +32,7 @@ import org.alfresco.repo.solr.Acl;
import org.alfresco.repo.solr.AclChangeSet;
import org.alfresco.repo.solr.NodeParameters;
import org.alfresco.repo.solr.Transaction;
import org.alfresco.service.namespace.QName;
/**
* DAO support for SOLR web scripts.
@@ -78,7 +79,8 @@ public interface SOLRDAO
* Get the nodes satisfying the constraints in nodeParameters
*
* @param nodeParameters set of constraints for which nodes to return
* @param shardPropertQName
* @return list of matching nodes
*/
public List<Node> getNodes(NodeParameters nodeParameters);
public List<Node> getNodes(NodeParameters nodeParameters, QName shardPropertQName);
}

24
source/java/org/alfresco/repo/domain/solr/ibatis/SOLRDAOImpl.java
View File

@@ -37,10 +37,12 @@ import org.alfresco.repo.domain.solr.AclEntity;
import org.alfresco.repo.domain.solr.NodeParametersEntity;
import org.alfresco.repo.domain.solr.SOLRDAO;
import org.alfresco.repo.domain.solr.SOLRTrackingParameters;
import org.alfresco.repo.search.impl.QueryParserUtils;
import org.alfresco.repo.solr.Acl;
import org.alfresco.repo.solr.AclChangeSet;
import org.alfresco.repo.solr.NodeParameters;
import org.alfresco.repo.solr.Transaction;
import org.alfresco.service.namespace.NamespaceService;
import org.alfresco.service.namespace.QName;
import org.alfresco.util.Pair;
import org.alfresco.util.PropertyCheck;
@@ -197,10 +199,30 @@ public class SOLRDAOImpl implements SOLRDAO
* {@inheritDoc}
*/
@SuppressWarnings("unchecked")
public List<Node> getNodes(NodeParameters nodeParameters)
public List<Node> getNodes(NodeParameters nodeParameters, QName shardPropertyQName)
{
NodeParametersEntity params = new NodeParametersEntity(nodeParameters, qnameDAO);
if(shardPropertyQName !=null)
{
if(shardPropertyQName.equals(ContentModel.PROP_CREATED))
{
params.setShardPropertyQNameId(-1L);
}
else if (shardPropertyQName.equals(ContentModel.PROP_MODIFIED))
{
params.setShardPropertyQNameId(-2L);
}
else
{
Pair<Long, QName> propertyQNamePair = qnameDAO.getQName(shardPropertyQName);
if(propertyQNamePair != null)
{
params.setShardPropertyQNameId(propertyQNamePair.getFirst());
}
}
}
if(nodeParameters.getMaxResults() != 0 && nodeParameters.getMaxResults() != Integer.MAX_VALUE)
{
return template.selectList(

2
source/java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguage.java
View File

@@ -331,7 +331,7 @@ public class DbOrIndexSwitchingQueryLanguage extends AbstractLuceneQueryLanguage
// TODO: setToTxnId(null) when SolrDAO behaviour is fixed.
nodeParameters.setToTxnId(Long.MAX_VALUE);
stopWatch.start("get changed nodes");
List<Node> changedNodeList = solrDao.getNodes(nodeParameters);
List<Node> changedNodeList = solrDao.getNodes(nodeParameters, null);
stopWatch.stop();
if (logger.isDebugEnabled())
{

12
source/java/org/alfresco/repo/solr/NodeParameters.java
View File

@@ -56,6 +56,8 @@ public class NodeParameters
private Set<QName> includeAspects;
private Set<QName> excludeAspects;
private String shardProperty;
public int getMaxResults()
{
return maxResults;
@@ -180,4 +182,14 @@ public class NodeParameters
{
this.excludeAspects = excludeAspects;
}
public String getShardProperty()
{
return this.shardProperty;
}
public void setShardProperty(String shardProperty)
{
this.shardProperty = shardProperty;
}
}

28
source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java
View File

@@ -53,6 +53,7 @@ import org.alfresco.repo.index.shard.ShardRegistry;
import org.alfresco.repo.index.shard.ShardState;
import org.alfresco.repo.search.AspectIndexFilter;
import org.alfresco.repo.search.TypeIndexFilter;
import org.alfresco.repo.search.impl.QueryParserUtils;
import org.alfresco.repo.security.authentication.AuthenticationUtil;
import org.alfresco.repo.tenant.TenantService;
import org.alfresco.service.cmr.dictionary.AspectDefinition;
@@ -70,10 +71,13 @@ import org.alfresco.service.cmr.repository.Path.ChildAssocElement;
import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
import org.alfresco.service.cmr.security.OwnableService;
import org.alfresco.service.cmr.security.PermissionService;
import org.alfresco.service.namespace.NamespaceService;
import org.alfresco.service.namespace.QName;
import org.alfresco.util.Pair;
import org.alfresco.util.PropertyCheck;
import com.sun.xml.txw2.NamespaceResolver;
/**
* Component providing data for SOLR tracking
*
@@ -95,6 +99,7 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
private TypeIndexFilter typeIndexFilter;
private AspectIndexFilter aspectIndexFilter;
private ShardRegistry shardRegistry;
private NamespaceService namespaceService;
@Override
@@ -177,6 +182,11 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
this.shardRegistry = shardRegistry;
}
public void setNamespaceService(NamespaceService namespaceService)
{
this.namespaceService = namespaceService;
}
/**
* Initialize
*/
@@ -193,6 +203,7 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
PropertyCheck.mandatory(this, "aclDAO", aclDAO);
PropertyCheck.mandatory(this, "typeIndexFilter", typeIndexFilter);
PropertyCheck.mandatory(this, "aspectIndexFilter", aspectIndexFilter);
PropertyCheck.mandatory(this, "namespaceService", namespaceService);
}
@Override
@@ -344,7 +355,22 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
{
if(enabled)
{
List<Node> nodes = solrDAO.getNodes(nodeParameters);
QName shardPropertQName = null;
if(nodeParameters.getShardProperty() != null)
{
PropertyDefinition pdef = QueryParserUtils.matchPropertyDefinition(NamespaceService.CONTENT_MODEL_1_0_URI, namespaceService, dictionaryService, nodeParameters.getShardProperty());
if(pdef == null)
{
throw new AlfrescoRuntimeException("Invalid shard property: "+nodeParameters.getShardProperty());
}
if((!pdef.getDataType().getName().equals(DataTypeDefinition.TEXT)) && (!pdef.getDataType().getName().equals(DataTypeDefinition.DATE)) && (!pdef.getDataType().getName().equals(DataTypeDefinition.DATETIME)))
{
throw new AlfrescoRuntimeException("Unsupported shard property type: "+(pdef.getDataType().getName() + " for " +nodeParameters.getShardProperty()));
}
shardPropertQName = pdef.getName();
}
List<Node> nodes = solrDAO.getNodes(nodeParameters, shardPropertQName);
for (Node node : nodes)
{

2
source/test-java/org/alfresco/repo/domain/solr/SOLRDAOTest.java
View File

@@ -102,7 +102,7 @@ public class SOLRDAOTest extends TestCase
@Override
public List<Node> execute() throws Throwable
{
return solrDAO.getNodes(nodeParameters);
return solrDAO.getNodes(nodeParameters, null);
}
}, true);
}

2
source/test-java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguageTest.java
View File

@@ -88,7 +88,7 @@ public class DbOrIndexSwitchingQueryLanguageTest
when(indexQueryLang.executeQuery(argThat(isSearchParamsSinceTxId(null)), eq(admLuceneSearcher))).thenReturn(indexResults);
when(indexResults.getLastIndexedTxId()).thenReturn(80L);
when(dbQueryLang.executeQuery(argThat(isSearchParamsSinceTxId(80L)), eq(admLuceneSearcher))).thenReturn(dbResults);
when(solrDAO.getNodes(argThat(isNodeParamsFromTxnId(81L)))).thenReturn(changedNodes);
when(solrDAO.getNodes(argThat(isNodeParamsFromTxnId(81L)), null)).thenReturn(changedNodes);
searchParameters.setQueryConsistency(QueryConsistency.HYBRID);

6
source/test-java/org/alfresco/repo/search/impl/solr/TestNode.java
View File

@@ -133,4 +133,10 @@ class TestNode implements Node
return null;
}
@Override
public String getShardKey()
{
return null;
}
}