SEARCH-224: Added tests for sharding by quarter or half a year

2026-09-16 18:12:56 +00:00 · 2016-11-11 09:31:26 +01:00
parent f375875a2d
commit 0de68ea1b2
5 changed files with 286 additions and 4 deletions
@@ -352,8 +352,14 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
        return jettyClients.get(DEFAULT_TEST_CORENAME);
    }

-    public void assertNodesPerShardGreaterThan(int count) throws Exception
+    public int assertNodesPerShardGreaterThan(int count) throws Exception
    {
+        return assertNodesPerShardGreaterThan(count, false);
+    }
+
+    public int assertNodesPerShardGreaterThan(int count, boolean ignoreZero) throws Exception
+    {
+        int shardHit = 0;
        List<SolrCore> cores = getJettyCores(jettyShards);
        Query query = new TermQuery(new Term(FIELD_DOC_TYPE, SolrInformationServer.DOC_TYPE_NODE));
        StringBuilder error = new StringBuilder();
@@ -365,10 +371,21 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
                refCounted = core.getSearcher();
                SolrIndexSearcher searcher = refCounted.get();
                TopDocs topDocs = searcher.search(query, 10);
+                if(topDocs.totalHits > 0)
+                {
+                    shardHit++;
+                }
                if(topDocs.totalHits < count)
                {
-                    error.append(" "+core.getName()+": ");
-                    error.append("Expected nodes per shard greater than "+count+" found "+topDocs.totalHits+" : "+query.toString());
+                    if (ignoreZero && topDocs.totalHits == 0)
+                    {
+                        log.info(core.getName()+": have zero hits ");
+                    }
+                    else
+                    {
+                        error.append(" "+core.getName()+": ");
+                        error.append("Expected nodes per shard greater than "+count+" found "+topDocs.totalHits+" : "+query.toString());
+                    }
                }
                log.info(core.getName()+": Hits "+topDocs.totalHits);
            }
@@ -382,6 +399,7 @@ public abstract class AbstractAlfrescoDistributedTest extends SolrTestCaseJ4
        {
            throw new Exception(error.toString());
        }
+        return shardHit;
    }

    public void assertCountAndColocation(Query query, int count) throws Exception
@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2005-2014 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.solr.tracker;
+
+import org.alfresco.model.ContentModel;
+import org.alfresco.repo.index.shard.ShardMethodEnum;
+import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
+import org.alfresco.solr.AbstractAlfrescoDistributedTest;
+import org.alfresco.solr.AlfrescoSolrDataModel;
+import org.alfresco.solr.client.*;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.LegacyNumericRangeQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.*;
+
+import static org.alfresco.solr.AlfrescoSolrUtils.*;
+
+/**
+ * Abstract sharding by date test
+ * @author Gethin James
+ */
+
+@SolrTestCaseJ4.SuppressSSL
+@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"})
+public abstract class DistributedDateAbstractSolrTrackerTest extends AbstractAlfrescoDistributedTest
+{
+    @Test
+    public void testDateMonth() throws Exception
+    {
+        Thread.sleep(12000);
+        putHandleDefaults();
+
+        int numAcls = 25;
+        AclChangeSet bulkAclChangeSet = getAclChangeSet(numAcls);
+
+        List<Acl> bulkAcls = new ArrayList();
+        List<AclReaders> bulkAclReaders = new ArrayList();
+
+
+        for (int i = 0; i < numAcls; i++) {
+            Acl bulkAcl = getAcl(bulkAclChangeSet);
+            bulkAcls.add(bulkAcl);
+            bulkAclReaders.add(getAclReaders(bulkAclChangeSet,
+                    bulkAcl,
+                    list("joel" + bulkAcl.getId()),
+                    list("phil" + bulkAcl.getId()),
+                    null));
+        }
+
+        indexAclChangeSet(bulkAclChangeSet,
+                bulkAcls,
+                bulkAclReaders);
+
+        int numNodes = 1000;
+        List<Node> nodes = new ArrayList();
+        List<NodeMetaData> nodeMetaDatas = new ArrayList();
+
+        Transaction bigTxn = getTransaction(0, numNodes);
+
+        Date[] dates = setupDates();
+
+        int[] counts = new int[dates.length];
+
+        for (int i = 0; i < numNodes; i++) {
+            int aclIndex = i % numAcls;
+            int dateIndex = i % dates.length;
+            String dateString = DefaultTypeConverter.INSTANCE.convert(String.class, dates[dateIndex]);
+            counts[dateIndex]++;
+            Node node = getNode(bigTxn, bulkAcls.get(aclIndex), Node.SolrApiNodeStatus.UPDATED);
+            node.setShardPropertyValue(dateString);
+            nodes.add(node);
+            NodeMetaData nodeMetaData = getNodeMetaData(node, bigTxn, bulkAcls.get(aclIndex), "mike", null, false);
+            nodeMetaData.getProperties().put(ContentModel.PROP_CREATED,
+                    new StringPropertyValue(dateString));
+
+            nodeMetaDatas.add(nodeMetaData);
+        }
+
+        indexTransaction(bigTxn, nodes, nodeMetaDatas);
+        waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), numNodes, 100000);
+
+        List<AlfrescoSolrDataModel.FieldInstance> fieldInstanceList = AlfrescoSolrDataModel.getInstance().getIndexedFieldNamesForProperty(MetadataTracker.getShardProperty("created")).getFields();
+        AlfrescoSolrDataModel.FieldInstance fieldInstance = fieldInstanceList.get(0);
+        String fieldName = fieldInstance.getField();
+
+        for (int i = 0; i < dates.length; i++) {
+            LegacyNumericRangeQuery query = LegacyNumericRangeQuery.newLongRange(fieldName, dates[i].getTime(), dates[i].getTime() + 1, true, false);
+            assertCountAndColocation(query, counts[i]);
+            assertCorrect(numNodes);
+        }
+    }
+
+    protected Date[] setupDates() {
+        Date[] dates = new Date[12];
+
+        Calendar cal = new GregorianCalendar();
+        for (int i = 0; i < dates.length; i++) {
+            cal.set(1980, i, 21);
+            dates[i] = cal.getTime();
+        }
+        return dates;
+    }
+
+    protected abstract Properties getShardMethod();
+    protected abstract void assertCorrect(int numNodes) throws Exception;
+}
+
@@ -62,7 +62,7 @@ import org.junit.Test;
 public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest
 {
    @Rule
-    public JettyServerRule jetty = new JettyServerRule(5,getShardMethod());
+    public JettyServerRule jetty = new JettyServerRule(this.getClass().getSimpleName(), 5, getShardMethod(), new String[]{DEFAULT_TEST_CORENAME});

    @Test
    public void testDateMonth() throws Exception
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2005-2014 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.solr.tracker;
+
+import org.alfresco.model.ContentModel;
+import org.alfresco.repo.index.shard.ShardMethodEnum;
+import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
+import org.alfresco.solr.AbstractAlfrescoDistributedTest;
+import org.alfresco.solr.AlfrescoSolrDataModel;
+import org.alfresco.solr.client.*;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.LegacyNumericRangeQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.*;
+
+import static org.alfresco.solr.AlfrescoSolrUtils.*;
+
+/**
+ * Tests sharding by quarter
+ * @author Gethin James
+ */
+
+@SolrTestCaseJ4.SuppressSSL
+@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"})
+public class DistributedDateQuarterAlfrescoSolrTrackerTest extends DistributedDateAbstractSolrTrackerTest
+{
+    @Rule
+    public JettyServerRule jetty = new JettyServerRule(this.getClass().getSimpleName(), 6, getShardMethod(), new String[]{DEFAULT_TEST_CORENAME});
+
+    @Override
+    protected void assertCorrect(int numNodes) throws Exception {
+        //We should expect roughly 25% on each of the 4 cores
+        int shardHits = assertNodesPerShardGreaterThan((int)((numNodes)*.22), true);
+        //We have 6 shards but we are sharding by quarter so only 4 of the shards should be used.
+        assertEquals(4, shardHits);
+    }
+
+    @Override
+    protected Properties getShardMethod()
+    {
+        Properties prop = new Properties();
+        prop.put("shard.method", ShardMethodEnum.DATE.toString());
+        prop.put("shard.date.grouping", "3");
+        return prop;
+    }
+}
+
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2005-2014 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.solr.tracker;
+
+import org.alfresco.model.ContentModel;
+import org.alfresco.repo.index.shard.ShardMethodEnum;
+import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
+import org.alfresco.solr.AbstractAlfrescoDistributedTest;
+import org.alfresco.solr.AlfrescoSolrDataModel;
+import org.alfresco.solr.client.*;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.LegacyNumericRangeQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.*;
+
+import static org.alfresco.solr.AlfrescoSolrUtils.*;
+
+/**
+ * Tests sharding by date, splits the year in 2
+ * @author Gethin James
+ */
+
+@SolrTestCaseJ4.SuppressSSL
+@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"})
+public class DistributedDateSplitYearAlfrescoSolrTrackerTest extends DistributedDateAbstractSolrTrackerTest
+{
+    @Rule
+    public JettyServerRule jetty = new JettyServerRule(this.getClass().getSimpleName(), 3, getShardMethod(), new String[]{DEFAULT_TEST_CORENAME});
+
+    @Override
+    protected void assertCorrect(int numNodes) throws Exception {
+        //We should expect roughly 50% on each of the 2 cores
+        int shardHits = assertNodesPerShardGreaterThan((int)((numNodes)*.48), true);
+        //We have 3 shards but we should be only be using 2
+        assertEquals(2, shardHits);
+    }
+
+    @Override
+    protected Properties getShardMethod()
+    {
+        Properties prop = new Properties();
+        prop.put("shard.method", ShardMethodEnum.DATE.toString());
+        prop.put("shard.date.grouping", "6");
+        return prop;
+    }
+}
+