Merged searchapi (5.2.1) to 5.2.N (5.2.1)

130022 gjames: Merged 5.2.N-AHIND (5.2.1) to searchapi (5.2.1)
      129926 ahind: SEARCH-31    SOLR 6 - Shard based on date - SOLR implementation
      - Support returning a date, datetime or string property to use for sharding


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/BRANCHES/DEV/5.2.N/root@130285 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/ibatis/org.hibernate.dialect.Dialect/solr-common-SqlMap.xml

@@ -30,6 +30,7 @@
         <result property="store.protocol" column="protocol" jdbcType="VARCHAR" javaType="java.lang.String"/>
         <result property="store.identifier" column="identifier" jdbcType="VARCHAR" javaType="java.lang.String"/>
         <result property="uuid" column="uuid" jdbcType="VARCHAR" javaType="java.lang.String"/>
+        <result property="shardKey" column="shard_key" jdbcType="VARCHAR" javaType="java.lang.String"/>
     </resultMap>
     
     <!-- SELECTS -->
@@ -119,12 +120,29 @@
             node.uuid           as uuid,
             node.acl_id         as acl_id,
             store.protocol      as protocol,
-            store.identifier    as identifier
+            store.identifier    as identifier,
+            <choose>
+               <when test="shardPropertyQNameId == null">
+                  null as shard_key
+               </when>
+               <when test="shardPropertyQNameId == -1">
+                  node.audit_created as shard_key
+               </when>
+               <when test="shardPropertyQNameId == -2">
+                  node.audit_modified as shard_key
+               </when>
+               <otherwise>
+                  shardkey.string_value as shard_key
+               </otherwise>
+            </choose>
         from
             alf_transaction txn
             join alf_node node on (txn.id = node.transaction_id)
             join alf_store store on (store.id = node.store_id)
             left outer join alf_node_properties np on (np.node_id = node.id and np.qname_id = #{originalIdPropQNameId})
+            <if test="shardPropertyQNameId != null and shardPropertyQNameId >= 0" >
+                left outer join alf_node_properties shardkey on (shardkey.node_id = node.id and shardkey.qname_id = #{shardPropertyQNameId})
+            </if>
         <where>
            <choose>
 	           <when test="transactionIds != null">

config/alfresco/subsystems/Search/common-search-context.xml

@@ -20,6 +20,7 @@
         </property>
         <property name="typeIndexFilter" ref="search.TypeIndexFilter" />
         <property name="aspectIndexFilter" ref="search.AspectIndexFilter" />
+        <property name="namespaceService" ref="namespaceService" />
     </bean>
 
     <!--  Ignore indexing by node type or by node aspects  -->

source/java/org/alfresco/repo/domain/node/Node.java

@@ -72,6 +72,8 @@ public interface Node extends NodeIdAndAclId
 
     public abstract TransactionEntity getTransaction();
 
-    public abstract AuditablePropertiesEntity getAuditableProperties();
+    public abstract AuditablePropertiesEntity getAuditableProperties();
+    
+    public abstract String getShardKey();
 
 }

source/java/org/alfresco/repo/domain/node/NodeEntity.java

@@ -54,7 +54,8 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
     private Long localeId;
     private Long aclId;
     private TransactionEntity transaction;
-    private AuditablePropertiesEntity auditableProperties;
+    private AuditablePropertiesEntity auditableProperties;
+    private String shardKey;
     
     /**
      * Required default constructor
@@ -89,7 +90,8 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
         this.localeId = node.getLocaleId();
         this.aclId = node.getAclId();
         this.transaction = node.getTransaction();
-        this.auditableProperties = node.getAuditableProperties();
+        this.auditableProperties = node.getAuditableProperties();
+        this.shardKey = node.getShardKey();
     }
       
     @Override
@@ -137,7 +139,8 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
           .append(", localeId=").append(localeId)
           .append(", aclId=").append(aclId)
           .append(", transaction=").append(transaction)
-          .append(", auditProps=").append(auditableProperties)
+          .append(", auditProps=").append(auditableProperties)
+          .append(", shardKey=").append(shardKey)
           .append("]");
         return sb.toString();
     }
@@ -331,5 +334,18 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
     {
         checkLock();
         this.auditableProperties = auditableProperties;
-    }
+    }
+
+    @Override
+    public String getShardKey()
+    {
+        return this.shardKey;
+    }
+
+    public synchronized void setShardKey(String shardKey)
+    {
+        this.shardKey = shardKey;
+    }
+    
+    
 }

source/java/org/alfresco/repo/domain/solr/NodeParametersEntity.java

@@ -48,7 +48,8 @@ public class NodeParametersEntity extends NodeParameters
     private List<Long> includeAspectIds;
     private List<Long> excludeAspectIds;
     
-    private Long originalIdPropQNameId;
+    private Long originalIdPropQNameId;
+    private Long shardPropertyQNameId;
     
     /**
      * Public constructor, but not generally useful
@@ -160,5 +161,17 @@ public class NodeParametersEntity extends NodeParameters
     public boolean isIncludeNodesTable()
     {
         return (getFromNodeId() != null || getToNodeId() != null || getIncludeTypeIds() != null || getExcludeTypeIds() != null || getIncludeAspectIds() != null || getExcludeAspectIds() != null);
-    }
+    }
+
+    public Long getShardPropertyQNameId()
+    {
+        return this.shardPropertyQNameId;
+    }
+
+    public void setShardPropertyQNameId(Long shardPropertyQNameId)
+    {
+        this.shardPropertyQNameId = shardPropertyQNameId;
+    }
+    
+    
 }

source/java/org/alfresco/repo/domain/solr/SOLRDAO.java

@@ -31,7 +31,8 @@ import org.alfresco.repo.domain.node.Node;
 import org.alfresco.repo.solr.Acl;
 import org.alfresco.repo.solr.AclChangeSet;
 import org.alfresco.repo.solr.NodeParameters;
-import org.alfresco.repo.solr.Transaction;
+import org.alfresco.repo.solr.Transaction;
+import org.alfresco.service.namespace.QName;
 
 /**
  * DAO support for SOLR web scripts.
@@ -77,8 +78,9 @@ public interface SOLRDAO
     /**
      * Get the nodes satisfying the constraints in nodeParameters
      * 
-     * @param nodeParameters set of constraints for which nodes to return
+     * @param nodeParameters set of constraints for which nodes to return
+     * @param shardPropertQName 
      * @return list of matching nodes
      */
-	public List<Node> getNodes(NodeParameters nodeParameters);
+	public List<Node> getNodes(NodeParameters nodeParameters, QName shardPropertQName);
 }

source/java/org/alfresco/repo/domain/solr/ibatis/SOLRDAOImpl.java

@@ -36,11 +36,13 @@ import org.alfresco.repo.domain.qname.QNameDAO;
 import org.alfresco.repo.domain.solr.AclEntity;
 import org.alfresco.repo.domain.solr.NodeParametersEntity;
 import org.alfresco.repo.domain.solr.SOLRDAO;
-import org.alfresco.repo.domain.solr.SOLRTrackingParameters;
+import org.alfresco.repo.domain.solr.SOLRTrackingParameters;
+import org.alfresco.repo.search.impl.QueryParserUtils;
 import org.alfresco.repo.solr.Acl;
 import org.alfresco.repo.solr.AclChangeSet;
 import org.alfresco.repo.solr.NodeParameters;
-import org.alfresco.repo.solr.Transaction;
+import org.alfresco.repo.solr.Transaction;
+import org.alfresco.service.namespace.NamespaceService;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.util.Pair;
 import org.alfresco.util.PropertyCheck;
@@ -95,8 +97,8 @@ public class SOLRDAOImpl implements SOLRDAO
 
         // We simulate an ID for the sys:deleted type
         Pair<Long, QName> deletedTypeQNamePair = qnameDAO.getQName(ContentModel.TYPE_DELETED);
-        Long deletedTypeQNameId = deletedTypeQNamePair == null ? -1L : deletedTypeQNamePair.getFirst();
+        Long deletedTypeQNameId = deletedTypeQNamePair == null ? -1L : deletedTypeQNamePair.getFirst();
-
+     
         SOLRTrackingParameters params = new SOLRTrackingParameters(deletedTypeQNameId);
         params.setFromIdInclusive(minAclChangeSetId);
         params.setFromCommitTimeInclusive(fromCommitTime);
@@ -197,9 +199,29 @@ public class SOLRDAOImpl implements SOLRDAO
      * {@inheritDoc}
      */
     @SuppressWarnings("unchecked")
-	public List<Node> getNodes(NodeParameters nodeParameters)
+    public List<Node> getNodes(NodeParameters nodeParameters, QName shardPropertyQName)
-	{
+    {
-	    NodeParametersEntity params = new NodeParametersEntity(nodeParameters, qnameDAO);
+        NodeParametersEntity params = new NodeParametersEntity(nodeParameters, qnameDAO);
+
+        if(shardPropertyQName !=null)
+        {
+            if(shardPropertyQName.equals(ContentModel.PROP_CREATED))
+            {
+                params.setShardPropertyQNameId(-1L);
+            }
+            else if (shardPropertyQName.equals(ContentModel.PROP_MODIFIED))
+            {
+                params.setShardPropertyQNameId(-2L);
+            }
+            else
+            {
+                Pair<Long, QName> propertyQNamePair = qnameDAO.getQName(shardPropertyQName);
+                if(propertyQNamePair != null)
+                {
+                    params.setShardPropertyQNameId(propertyQNamePair.getFirst());
+                }
+            }
+        }
 
 	    if(nodeParameters.getMaxResults() != 0 && nodeParameters.getMaxResults() != Integer.MAX_VALUE)
 	    {

source/java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguage.java

@@ -331,7 +331,7 @@ public class DbOrIndexSwitchingQueryLanguage extends AbstractLuceneQueryLanguage
         // TODO: setToTxnId(null) when SolrDAO behaviour is fixed.
         nodeParameters.setToTxnId(Long.MAX_VALUE);
         stopWatch.start("get changed nodes");
-        List<Node> changedNodeList = solrDao.getNodes(nodeParameters);
+        List<Node> changedNodeList = solrDao.getNodes(nodeParameters, null);
         stopWatch.stop();
         if (logger.isDebugEnabled())
         {

source/java/org/alfresco/repo/solr/NodeParameters.java

@@ -54,7 +54,9 @@ public class NodeParameters
     private Set<QName> excludeNodeTypes;
     
     private Set<QName> includeAspects;
-    private Set<QName> excludeAspects;
+    private Set<QName> excludeAspects;
+    
+    private String shardProperty;
     
     public int getMaxResults()
     {
@@ -179,5 +181,15 @@ public class NodeParameters
     public void setExcludeAspects(Set<QName> excludeAspects)
     {
         this.excludeAspects = excludeAspects;
+    }
+
+    public String getShardProperty()
+    {
+        return this.shardProperty;
+    }
+
+    public void setShardProperty(String shardProperty)
+    {
+        this.shardProperty = shardProperty;
     }
 }

source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java

@@ -52,7 +52,8 @@ import org.alfresco.repo.domain.solr.SOLRDAO;
 import org.alfresco.repo.index.shard.ShardRegistry;
 import org.alfresco.repo.index.shard.ShardState;
 import org.alfresco.repo.search.AspectIndexFilter;
-import org.alfresco.repo.search.TypeIndexFilter;
+import org.alfresco.repo.search.TypeIndexFilter;
+import org.alfresco.repo.search.impl.QueryParserUtils;
 import org.alfresco.repo.security.authentication.AuthenticationUtil;
 import org.alfresco.repo.tenant.TenantService;
 import org.alfresco.service.cmr.dictionary.AspectDefinition;
@@ -69,10 +70,13 @@ import org.alfresco.service.cmr.repository.Path;
 import org.alfresco.service.cmr.repository.Path.ChildAssocElement;
 import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
 import org.alfresco.service.cmr.security.OwnableService;
-import org.alfresco.service.cmr.security.PermissionService;
+import org.alfresco.service.cmr.security.PermissionService;
+import org.alfresco.service.namespace.NamespaceService;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.util.Pair;
-import org.alfresco.util.PropertyCheck;
+import org.alfresco.util.PropertyCheck;
+
+import com.sun.xml.txw2.NamespaceResolver;
 
 /**
  * Component providing data for SOLR tracking
@@ -94,7 +98,8 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
     private boolean cacheAncestors =true;
     private TypeIndexFilter typeIndexFilter;
     private AspectIndexFilter aspectIndexFilter;
-    private ShardRegistry shardRegistry;
+    private ShardRegistry shardRegistry;
+    private NamespaceService namespaceService;
     
     
     @Override
@@ -177,6 +182,11 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
         this.shardRegistry = shardRegistry;
     }
 
+    public void setNamespaceService(NamespaceService namespaceService)
+    {
+        this.namespaceService = namespaceService;
+    }
+
     /**
      * Initialize
      */    
@@ -192,7 +202,8 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
         PropertyCheck.mandatory(this, "dictionaryDAO", dictionaryDAO);
         PropertyCheck.mandatory(this, "aclDAO", aclDAO);
         PropertyCheck.mandatory(this, "typeIndexFilter", typeIndexFilter);
-        PropertyCheck.mandatory(this, "aspectIndexFilter", aspectIndexFilter);
+        PropertyCheck.mandatory(this, "aspectIndexFilter", aspectIndexFilter);
+        PropertyCheck.mandatory(this, "namespaceService", namespaceService);
     }
     
     @Override
@@ -343,8 +354,23 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
 	public void getNodes(NodeParameters nodeParameters, NodeQueryCallback callback)
 	{
 	    if(enabled)
-	    {
+	    {
-	        List<Node> nodes = solrDAO.getNodes(nodeParameters);
+	        QName shardPropertQName = null;
+	        if(nodeParameters.getShardProperty() != null)
+	        {
+	            PropertyDefinition pdef = QueryParserUtils.matchPropertyDefinition(NamespaceService.CONTENT_MODEL_1_0_URI, namespaceService, dictionaryService, nodeParameters.getShardProperty());
+	            if(pdef == null)
+	            {
+	                throw new AlfrescoRuntimeException("Invalid shard property: "+nodeParameters.getShardProperty());
+	            }
+	            if((!pdef.getDataType().getName().equals(DataTypeDefinition.TEXT)) && (!pdef.getDataType().getName().equals(DataTypeDefinition.DATE))  && (!pdef.getDataType().getName().equals(DataTypeDefinition.DATETIME)))
+	            {
+	                throw new AlfrescoRuntimeException("Unsupported shard property type: "+(pdef.getDataType().getName() + " for " +nodeParameters.getShardProperty()));
+	            }
+	            shardPropertQName = pdef.getName();
+	        }
+	        
+	        List<Node> nodes = solrDAO.getNodes(nodeParameters, shardPropertQName);
 
 	        for (Node node : nodes)
 	        {

source/test-java/org/alfresco/repo/domain/solr/SOLRDAOTest.java

@@ -102,7 +102,7 @@ public class SOLRDAOTest extends TestCase
             @Override
             public List<Node> execute() throws Throwable
             {
-                return solrDAO.getNodes(nodeParameters);
+                return solrDAO.getNodes(nodeParameters, null);
             }
         }, true);
     }

source/test-java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguageTest.java

@@ -88,7 +88,7 @@ public class DbOrIndexSwitchingQueryLanguageTest
         when(indexQueryLang.executeQuery(argThat(isSearchParamsSinceTxId(null)), eq(admLuceneSearcher))).thenReturn(indexResults);
         when(indexResults.getLastIndexedTxId()).thenReturn(80L);
         when(dbQueryLang.executeQuery(argThat(isSearchParamsSinceTxId(80L)), eq(admLuceneSearcher))).thenReturn(dbResults);
-        when(solrDAO.getNodes(argThat(isNodeParamsFromTxnId(81L)))).thenReturn(changedNodes);
+        when(solrDAO.getNodes(argThat(isNodeParamsFromTxnId(81L)), null)).thenReturn(changedNodes);
         
         searchParameters.setQueryConsistency(QueryConsistency.HYBRID);
         

source/test-java/org/alfresco/repo/search/impl/solr/TestNode.java

@@ -131,6 +131,12 @@ class TestNode implements Node
     public AuditablePropertiesEntity getAuditableProperties()
     {
         return null;
+    }
+
+    @Override
+    public String getShardKey()
+    {
+        return null;
     }
 
 }