Merge branch 'feature/SEARCH-506' into 'develop'

Feature/search 506 Explicit shard routing See merge request !11
2026-09-16 18:12:56 +00:00 · 2017-08-16 08:54:19 +01:00
parent 859647ee60 61b3d646c6
commit 0690f00b73
4 changed files with 221 additions and 1 deletions
@@ -19,6 +19,8 @@
 package org.alfresco.solr.tracker;

 import org.alfresco.repo.index.shard.ShardMethodEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;

 import java.util.Properties;
 import java.util.Set;
@@ -29,26 +31,38 @@ import java.util.Set;

 public class DocRouterFactory
 {
+    protected final static Logger log = LoggerFactory.getLogger(DocRouterFactory.class);
+
    public static DocRouter getRouter(Properties properties, ShardMethodEnum method) {

        switch(method) {
            case DB_ID:
+                log.info("Sharding via DB_ID");
                return new DBIDRouter();
            case DB_ID_RANGE:
                String range = properties.getProperty("shard.range");
                String[] rangeParts = range.split("-");
                long startRange = Long.parseLong(rangeParts[0].trim());
                long endRange = Long.parseLong(rangeParts[1].trim());
+                log.info("Sharding via DB_ID_RANGE");
                return new DBIDRangeRouter(startRange, endRange);
            case ACL_ID:
+                log.info("Sharding via ACL_ID");
                return new ACLIDMurmurRouter();
            case MOD_ACL_ID:
+                log.info("Sharding via MOD_ACL_ID");
                return new ACLIDModRouter();
            case DATE:
+                log.info("Sharding via DATE");
                return new DateMonthRouter(properties.getProperty("shard.date.grouping", "1"));
            case PROPERTY:
+                log.info("Sharding via PROPERTY");
                return new PropertyRouter(properties.getProperty("shard.regex", ""));
+            case EXPLICIT_ID:
+                log.info("Sharding via EXPLICIT_ID");
+                return new ExplicitRouter();
            default:
+                log.info("Sharding via DB_ID (default)");
                return new DBIDRouter();
        }
    }
@@ -0,0 +1,59 @@
+package org.alfresco.solr.tracker;
+
+import org.alfresco.solr.client.Acl;
+import org.alfresco.solr.client.Node;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Routes a document only if the shardInstance matches the provided shardId
+ */
+public class ExplicitRouter implements DocRouter {
+
+    protected final static Logger log = LoggerFactory.getLogger(ExplicitRouter.class);
+
+    public ExplicitRouter() {
+    }
+
+    @Override
+    public boolean routeAcl(int shardCount, int shardInstance, Acl acl) {
+        //all acls go to all shards.
+        return true;
+    }
+
+    @Override
+    public boolean routeNode(int shardCount, int shardInstance, Node node) {
+
+        if(shardCount <= 1)
+        {
+            return true;
+        }
+
+        String shardBy = node.getShardPropertyValue();
+
+        if (shardBy != null && !shardBy.isEmpty())
+        {
+            try
+            {
+                int shardid = Integer.parseInt(shardBy);
+                return shardid == shardInstance;
+            }
+            catch (NumberFormatException e)
+            {
+                if (log.isDebugEnabled())
+                {
+                    log.debug("EXPLICIT_ID routing specified but failed to parse a shard property value of "+shardBy+" for node "+node.getNodeRef());
+                }
+            }
+        }
+        else
+        {
+            if (log.isDebugEnabled())
+            {
+                log.debug("EXPLICIT_ID routing specified but no shard id property found for node "+node.getNodeRef());
+            }
+        }
+
+        return false;
+    }
+}
@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2005-2016 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.solr.tracker;
+
+import com.carrotsearch.randomizedtesting.RandomizedContext;
+import org.alfresco.model.ContentModel;
+import org.alfresco.repo.index.shard.ShardMethodEnum;
+import org.alfresco.solr.AbstractAlfrescoDistributedTest;
+import org.alfresco.solr.SolrInformationServer;
+import org.alfresco.solr.client.*;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.core.SolrCore;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.*;
+
+import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE;
+import static org.alfresco.solr.AlfrescoSolrUtils.*;
+
+/**
+ * Test Routes based on an explicit shard
+ *
+ * @author Gethin James
+ */
+@SolrTestCaseJ4.SuppressSSL
+@SolrTestCaseJ4.SuppressObjectReleaseTracker (bugUrl = "RAMDirectory")
+@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"})
+public class DistributedExplicitShardRoutingTrackerTest extends AbstractAlfrescoDistributedTest
+{
+    @Rule
+    public JettyServerRule jetty = new JettyServerRule(this.getClass().getSimpleName(), 3, getProperties(), new String[]{DEFAULT_TEST_CORENAME});
+
+    @Test
+    public void testShardId() throws Exception
+    {
+        putHandleDefaults();
+
+        int numAcls = 25;
+        AclChangeSet bulkAclChangeSet = getAclChangeSet(numAcls);
+
+        List<Acl> bulkAcls = new ArrayList();
+        List<AclReaders> bulkAclReaders = new ArrayList();
+
+
+        for (int i = 0; i < numAcls; i++) {
+            Acl bulkAcl = getAcl(bulkAclChangeSet);
+            bulkAcls.add(bulkAcl);
+            bulkAclReaders.add(getAclReaders(bulkAclChangeSet,
+                    bulkAcl,
+                    list("king" + bulkAcl.getId()),
+                    list("king" + bulkAcl.getId()),
+                    null));
+        }
+
+        indexAclChangeSet(bulkAclChangeSet,
+                bulkAcls,
+                bulkAclReaders);
+
+        int numNodes = 1000;
+        List<Node> nodes = new ArrayList();
+        List<NodeMetaData> nodeMetaDatas = new ArrayList();
+
+        Transaction bigTxn = getTransaction(0, numNodes);
+
+        for (int i = 0; i < numNodes; i++) {
+            int aclIndex = i % numAcls;
+            Node node = getNode(bigTxn, bulkAcls.get(aclIndex), Node.SolrApiNodeStatus.UPDATED);
+            nodes.add(node);
+            NodeMetaData nodeMetaData = getNodeMetaData(node, bigTxn, bulkAcls.get(aclIndex), "king", null, false);
+            boolean even = i % 2 == 0; // if its even put it on shard 1 otherwise put it on shard 0.
+            node.setShardPropertyValue(even?"1":"0");
+            nodeMetaDatas.add(nodeMetaData);
+        }
+
+        //Add a node that won't get indexed
+        Node node = getNode(bigTxn, bulkAcls.get(1), Node.SolrApiNodeStatus.UPDATED);
+        nodes.add(node);
+        NodeMetaData nodeMetaData = getNodeMetaData(node, bigTxn, bulkAcls.get(1), "king", null, false);
+        node.setShardPropertyValue("node YOU DON'T");
+        nodeMetaDatas.add(nodeMetaData);
+
+        //Add a node that won't get indexed
+        node = getNode(bigTxn, bulkAcls.get(2), Node.SolrApiNodeStatus.UPDATED);
+        nodes.add(node);
+        nodeMetaData = getNodeMetaData(node, bigTxn, bulkAcls.get(2), "king", null, false);
+        //Don't set the Share Property but add it anyway
+        nodeMetaDatas.add(nodeMetaData);
+
+        indexTransaction(bigTxn, nodes, nodeMetaDatas);
+
+        Query contentQuery = new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world"));
+        Query aclQuery = new TermQuery(new Term(FIELD_DOC_TYPE, SolrInformationServer.DOC_TYPE_ACL));
+        List<SolrCore> shards = getJettyCores(jettyShards);
+        long begin = System.currentTimeMillis();
+
+        for (SolrCore core : shards)
+        {
+            switch (core.getName())
+            {
+                case "shard0":
+                case "shard1":
+                    waitForDocCountCore(core, contentQuery, 500, 50000, begin);
+                    break;
+                default:
+                    //ignore other shards because we will check below
+            }
+        }
+
+        //lets make sure the other nodes don't have any.
+         assertShardCount(2, contentQuery, 0);
+
+        //Acls go to all cores
+        waitForDocCountAllCores(aclQuery, numAcls, 20000);
+    }
+
+    protected Properties getProperties()
+    {
+        Properties prop = new Properties();
+        prop.put("shard.method", ShardMethodEnum.EXPLICIT_ID.toString());
+        //Normally this would be used by the Solr client which will automatically add the property to the node.shardPropertyValue
+        //For testing this doesn't work like that so I setShardPropertyValue explicitly above.
+        prop.put("shard.key", ContentModel.PROP_SKYPE.toString());
+        return prop;
+    }
+}
@@ -12,7 +12,7 @@
    <name>Alfresco Solr Search parent</name>
    <properties>
        <solr.version>6.6.0</solr.version>
-        <alfresco-solr.version>5.2.2-SNAPSHOT</alfresco-solr.version>
+        <alfresco-solr.version>5.3-SNAPSHOT</alfresco-solr.version>
    </properties>
    <distributionManagement>
        <repository>