Merged searchapi (5.2.1) to 5.2.N (5.2.1)

130022 gjames: Merged 5.2.N-AHIND (5.2.1) to searchapi (5.2.1)
      129926 ahind: SEARCH-31    SOLR 6 - Shard based on date - SOLR implementation
      - Support returning a date, datetime or string property to use for sharding


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/BRANCHES/DEV/5.2.N/root@130285 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/ibatis/org.hibernate.dialect.Dialect/solr-common-SqlMap.xml

@@ -30,6 +30,7 @@
         <result property="store.protocol" column="protocol" jdbcType="VARCHAR" javaType="java.lang.String"/>
         <result property="store.identifier" column="identifier" jdbcType="VARCHAR" javaType="java.lang.String"/>
         <result property="uuid" column="uuid" jdbcType="VARCHAR" javaType="java.lang.String"/>
+        <result property="shardKey" column="shard_key" jdbcType="VARCHAR" javaType="java.lang.String"/>
     </resultMap>
     
     <!-- SELECTS -->
@@ -119,12 +120,29 @@
             node.uuid           as uuid,
             node.acl_id         as acl_id,
             store.protocol      as protocol,
-            store.identifier    as identifier
+            store.identifier    as identifier,
+            <choose>
+               <when test="shardPropertyQNameId == null">
+                  null as shard_key
+               </when>
+               <when test="shardPropertyQNameId == -1">
+                  node.audit_created as shard_key
+               </when>
+               <when test="shardPropertyQNameId == -2">
+                  node.audit_modified as shard_key
+               </when>
+               <otherwise>
+                  shardkey.string_value as shard_key
+               </otherwise>
+            </choose>
         from
             alf_transaction txn
             join alf_node node on (txn.id = node.transaction_id)
             join alf_store store on (store.id = node.store_id)
             left outer join alf_node_properties np on (np.node_id = node.id and np.qname_id = #{originalIdPropQNameId})
+            <if test="shardPropertyQNameId != null and shardPropertyQNameId >= 0" >
+                left outer join alf_node_properties shardkey on (shardkey.node_id = node.id and shardkey.qname_id = #{shardPropertyQNameId})
+            </if>
         <where>
            <choose>
 	           <when test="transactionIds != null">

config/alfresco/subsystems/Search/common-search-context.xml

@@ -20,6 +20,7 @@
         </property>
         <property name="typeIndexFilter" ref="search.TypeIndexFilter" />
         <property name="aspectIndexFilter" ref="search.AspectIndexFilter" />
+        <property name="namespaceService" ref="namespaceService" />
     </bean>
 
     <!--  Ignore indexing by node type or by node aspects  -->

source/java/org/alfresco/repo/domain/node/Node.java

@@ -74,4 +74,6 @@ public interface Node extends NodeIdAndAclId
 
     public abstract AuditablePropertiesEntity getAuditableProperties();
     
+    public abstract String getShardKey();
+
 }

source/java/org/alfresco/repo/domain/node/NodeEntity.java

@@ -55,6 +55,7 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
     private Long aclId;
     private TransactionEntity transaction;
     private AuditablePropertiesEntity auditableProperties;
+    private String shardKey;
     
     /**
      * Required default constructor
@@ -90,6 +91,7 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
         this.aclId = node.getAclId();
         this.transaction = node.getTransaction();
         this.auditableProperties = node.getAuditableProperties();
+        this.shardKey = node.getShardKey();
     }
       
     @Override
@@ -138,6 +140,7 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
           .append(", aclId=").append(aclId)
           .append(", transaction=").append(transaction)
           .append(", auditProps=").append(auditableProperties)
+          .append(", shardKey=").append(shardKey)
           .append("]");
         return sb.toString();
     }
@@ -332,4 +335,17 @@ public class NodeEntity implements Node, PermissionCheckValue, Serializable, Clo
         checkLock();
         this.auditableProperties = auditableProperties;
     }
+
+    @Override
+    public String getShardKey()
+    {
+        return this.shardKey;
+    }
+
+    public synchronized void setShardKey(String shardKey)
+    {
+        this.shardKey = shardKey;
+    }
+    
+    
 }

source/java/org/alfresco/repo/domain/solr/NodeParametersEntity.java

@@ -49,6 +49,7 @@ public class NodeParametersEntity extends NodeParameters
     private List<Long> excludeAspectIds;
     
     private Long originalIdPropQNameId;
+    private Long shardPropertyQNameId;
     
     /**
      * Public constructor, but not generally useful
@@ -161,4 +162,16 @@ public class NodeParametersEntity extends NodeParameters
     {
         return (getFromNodeId() != null || getToNodeId() != null || getIncludeTypeIds() != null || getExcludeTypeIds() != null || getIncludeAspectIds() != null || getExcludeAspectIds() != null);
     }
+
+    public Long getShardPropertyQNameId()
+    {
+        return this.shardPropertyQNameId;
+    }
+
+    public void setShardPropertyQNameId(Long shardPropertyQNameId)
+    {
+        this.shardPropertyQNameId = shardPropertyQNameId;
+    }
+    
+    
 }

source/java/org/alfresco/repo/domain/solr/SOLRDAO.java

@@ -32,6 +32,7 @@ import org.alfresco.repo.solr.Acl;
 import org.alfresco.repo.solr.AclChangeSet;
 import org.alfresco.repo.solr.NodeParameters;
 import org.alfresco.repo.solr.Transaction;
+import org.alfresco.service.namespace.QName;
 
 /**
  * DAO support for SOLR web scripts.
@@ -78,7 +79,8 @@ public interface SOLRDAO
      * Get the nodes satisfying the constraints in nodeParameters
      * 
      * @param nodeParameters set of constraints for which nodes to return
+     * @param shardPropertQName 
      * @return list of matching nodes
      */
-	public List<Node> getNodes(NodeParameters nodeParameters);
+	public List<Node> getNodes(NodeParameters nodeParameters, QName shardPropertQName);
 }

source/java/org/alfresco/repo/domain/solr/ibatis/SOLRDAOImpl.java

@@ -37,10 +37,12 @@ import org.alfresco.repo.domain.solr.AclEntity;
 import org.alfresco.repo.domain.solr.NodeParametersEntity;
 import org.alfresco.repo.domain.solr.SOLRDAO;
 import org.alfresco.repo.domain.solr.SOLRTrackingParameters;
+import org.alfresco.repo.search.impl.QueryParserUtils;
 import org.alfresco.repo.solr.Acl;
 import org.alfresco.repo.solr.AclChangeSet;
 import org.alfresco.repo.solr.NodeParameters;
 import org.alfresco.repo.solr.Transaction;
+import org.alfresco.service.namespace.NamespaceService;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.util.Pair;
 import org.alfresco.util.PropertyCheck;
@@ -197,10 +199,30 @@ public class SOLRDAOImpl implements SOLRDAO
      * {@inheritDoc}
      */
     @SuppressWarnings("unchecked")
-	public List<Node> getNodes(NodeParameters nodeParameters)
+    public List<Node> getNodes(NodeParameters nodeParameters, QName shardPropertyQName)
     {
         NodeParametersEntity params = new NodeParametersEntity(nodeParameters, qnameDAO);
 
+        if(shardPropertyQName !=null)
+        {
+            if(shardPropertyQName.equals(ContentModel.PROP_CREATED))
+            {
+                params.setShardPropertyQNameId(-1L);
+            }
+            else if (shardPropertyQName.equals(ContentModel.PROP_MODIFIED))
+            {
+                params.setShardPropertyQNameId(-2L);
+            }
+            else
+            {
+                Pair<Long, QName> propertyQNamePair = qnameDAO.getQName(shardPropertyQName);
+                if(propertyQNamePair != null)
+                {
+                    params.setShardPropertyQNameId(propertyQNamePair.getFirst());
+                }
+            }
+        }
+
 	    if(nodeParameters.getMaxResults() != 0 && nodeParameters.getMaxResults() != Integer.MAX_VALUE)
 	    {
 	        return template.selectList(

source/java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguage.java

@@ -331,7 +331,7 @@ public class DbOrIndexSwitchingQueryLanguage extends AbstractLuceneQueryLanguage
         // TODO: setToTxnId(null) when SolrDAO behaviour is fixed.
         nodeParameters.setToTxnId(Long.MAX_VALUE);
         stopWatch.start("get changed nodes");
-        List<Node> changedNodeList = solrDao.getNodes(nodeParameters);
+        List<Node> changedNodeList = solrDao.getNodes(nodeParameters, null);
         stopWatch.stop();
         if (logger.isDebugEnabled())
         {

source/java/org/alfresco/repo/solr/NodeParameters.java

@@ -56,6 +56,8 @@ public class NodeParameters
     private Set<QName> includeAspects;
     private Set<QName> excludeAspects;
     
+    private String shardProperty;
+    
     public int getMaxResults()
     {
         return maxResults;
@@ -180,4 +182,14 @@ public class NodeParameters
     {
         this.excludeAspects = excludeAspects;
     }
+
+    public String getShardProperty()
+    {
+        return this.shardProperty;
+    }
+
+    public void setShardProperty(String shardProperty)
+    {
+        this.shardProperty = shardProperty;
+    }
 }

source/java/org/alfresco/repo/solr/SOLRTrackingComponentImpl.java

@@ -53,6 +53,7 @@ import org.alfresco.repo.index.shard.ShardRegistry;
 import org.alfresco.repo.index.shard.ShardState;
 import org.alfresco.repo.search.AspectIndexFilter;
 import org.alfresco.repo.search.TypeIndexFilter;
+import org.alfresco.repo.search.impl.QueryParserUtils;
 import org.alfresco.repo.security.authentication.AuthenticationUtil;
 import org.alfresco.repo.tenant.TenantService;
 import org.alfresco.service.cmr.dictionary.AspectDefinition;
@@ -70,10 +71,13 @@ import org.alfresco.service.cmr.repository.Path.ChildAssocElement;
 import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
 import org.alfresco.service.cmr.security.OwnableService;
 import org.alfresco.service.cmr.security.PermissionService;
+import org.alfresco.service.namespace.NamespaceService;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.util.Pair;
 import org.alfresco.util.PropertyCheck;
 
+import com.sun.xml.txw2.NamespaceResolver;
+
 /**
  * Component providing data for SOLR tracking
  * 
@@ -95,6 +99,7 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
     private TypeIndexFilter typeIndexFilter;
     private AspectIndexFilter aspectIndexFilter;
     private ShardRegistry shardRegistry;
+    private NamespaceService namespaceService;
     
     
     @Override
@@ -177,6 +182,11 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
         this.shardRegistry = shardRegistry;
     }
 
+    public void setNamespaceService(NamespaceService namespaceService)
+    {
+        this.namespaceService = namespaceService;
+    }
+
     /**
      * Initialize
      */    
@@ -193,6 +203,7 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
         PropertyCheck.mandatory(this, "aclDAO", aclDAO);
         PropertyCheck.mandatory(this, "typeIndexFilter", typeIndexFilter);
         PropertyCheck.mandatory(this, "aspectIndexFilter", aspectIndexFilter);
+        PropertyCheck.mandatory(this, "namespaceService", namespaceService);
     }
     
     @Override
@@ -344,7 +355,22 @@ public class SOLRTrackingComponentImpl implements SOLRTrackingComponent
 	{
 	    if(enabled)
 	    {
-	        List<Node> nodes = solrDAO.getNodes(nodeParameters);
+	        QName shardPropertQName = null;
+	        if(nodeParameters.getShardProperty() != null)
+	        {
+	            PropertyDefinition pdef = QueryParserUtils.matchPropertyDefinition(NamespaceService.CONTENT_MODEL_1_0_URI, namespaceService, dictionaryService, nodeParameters.getShardProperty());
+	            if(pdef == null)
+	            {
+	                throw new AlfrescoRuntimeException("Invalid shard property: "+nodeParameters.getShardProperty());
+	            }
+	            if((!pdef.getDataType().getName().equals(DataTypeDefinition.TEXT)) && (!pdef.getDataType().getName().equals(DataTypeDefinition.DATE))  && (!pdef.getDataType().getName().equals(DataTypeDefinition.DATETIME)))
+	            {
+	                throw new AlfrescoRuntimeException("Unsupported shard property type: "+(pdef.getDataType().getName() + " for " +nodeParameters.getShardProperty()));
+	            }
+	            shardPropertQName = pdef.getName();
+	        }
+	        
+	        List<Node> nodes = solrDAO.getNodes(nodeParameters, shardPropertQName);
 
 	        for (Node node : nodes)
 	        {

source/test-java/org/alfresco/repo/domain/solr/SOLRDAOTest.java

@@ -102,7 +102,7 @@ public class SOLRDAOTest extends TestCase
             @Override
             public List<Node> execute() throws Throwable
             {
-                return solrDAO.getNodes(nodeParameters);
+                return solrDAO.getNodes(nodeParameters, null);
             }
         }, true);
     }

source/test-java/org/alfresco/repo/search/impl/solr/DbOrIndexSwitchingQueryLanguageTest.java

@@ -88,7 +88,7 @@ public class DbOrIndexSwitchingQueryLanguageTest
         when(indexQueryLang.executeQuery(argThat(isSearchParamsSinceTxId(null)), eq(admLuceneSearcher))).thenReturn(indexResults);
         when(indexResults.getLastIndexedTxId()).thenReturn(80L);
         when(dbQueryLang.executeQuery(argThat(isSearchParamsSinceTxId(80L)), eq(admLuceneSearcher))).thenReturn(dbResults);
-        when(solrDAO.getNodes(argThat(isNodeParamsFromTxnId(81L)))).thenReturn(changedNodes);
+        when(solrDAO.getNodes(argThat(isNodeParamsFromTxnId(81L)), null)).thenReturn(changedNodes);
         
         searchParameters.setQueryConsistency(QueryConsistency.HYBRID);
         

source/test-java/org/alfresco/repo/search/impl/solr/TestNode.java

@@ -133,4 +133,10 @@ class TestNode implements Node
         return null;
     }
 
+    @Override
+    public String getShardKey()
+    {
+        return null;
+    }
+
 }