Search-77, added ability to configure tracker schedule to content, metadata and acl

2026-09-16 18:12:56 +00:00 · 2017-07-06 13:09:12 +01:00
parent 6e90828975
commit 17c0acc5f8
12 changed files with 170 additions and 38 deletions
@@ -20,13 +20,10 @@
 package org.alfresco.solr.lifecycle;

 import org.alfresco.solr.AlfrescoCoreAdminHandler;
-import org.apache.solr.core.*;
-import org.apache.solr.handler.admin.CoreAdminHandler;
+import org.apache.solr.core.AbstractSolrEventListener;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
 import org.apache.solr.search.SolrIndexSearcher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.invoke.MethodHandles;

 /**
 * Listens for the first searcher to be created for a core and registers the trackers
@@ -18,22 +18,37 @@
 */
 package org.alfresco.solr.lifecycle;

-import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService;
-import org.alfresco.solr.*;
-import org.alfresco.solr.client.SOLRAPIClient;
-import org.alfresco.solr.client.SOLRAPIClientFactory;
-import org.alfresco.solr.content.SolrContentStore;
-import org.alfresco.solr.tracker.*;
-import org.apache.solr.core.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Properties;

+import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService;
+import org.alfresco.solr.AlfrescoCoreAdminHandler;
+import org.alfresco.solr.AlfrescoSolrDataModel;
+import org.alfresco.solr.SolrInformationServer;
+import org.alfresco.solr.SolrKeyResourceLoader;
+import org.alfresco.solr.client.SOLRAPIClient;
+import org.alfresco.solr.client.SOLRAPIClientFactory;
+import org.alfresco.solr.content.SolrContentStore;
+import org.alfresco.solr.tracker.AclTracker;
+import org.alfresco.solr.tracker.CascadeTracker;
+import org.alfresco.solr.tracker.CommitTracker;
+import org.alfresco.solr.tracker.ContentTracker;
+import org.alfresco.solr.tracker.MetadataTracker;
+import org.alfresco.solr.tracker.ModelTracker;
+import org.alfresco.solr.tracker.SolrTrackerScheduler;
+import org.alfresco.solr.tracker.Tracker;
+import org.alfresco.solr.tracker.TrackerRegistry;
+import org.apache.solr.core.CloseHook;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptorDecorator;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
 * Deals with core registration when the core is loaded.
 *
@@ -136,7 +151,7 @@ public class SolrCoreLoadRegistration {
     * @return A list of trackers
     */
    private static List<Tracker> createTrackers(String coreName, TrackerRegistry trackerRegistry, Properties props, SolrTrackerScheduler scheduler, SOLRAPIClient repositoryClient, SolrInformationServer srv) {
-        List<Tracker> trackers = new ArrayList();
+        List<Tracker> trackers = new ArrayList<Tracker>();

        AclTracker aclTracker = new AclTracker(props, repositoryClient, coreName, srv);
        trackerRegistry.register(coreName, aclTracker);
@@ -305,6 +305,11 @@ public abstract class AbstractTracker implements Tracker
    {
        return alfrescoVersion;
    }
+
+    public Properties getProps()
+    {
+        return props;
+    }
    
    
 }
@@ -62,8 +62,6 @@ public class AclTracker extends AbstractTracker
    private int changeSetAclsBatchSize = DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE;
    private int aclBatchSize = DEFAULT_ACL_BATCH_SIZE;

-    private int  maxAclChangeSetDocumentIdCacheSize = 700000;
-
    private ConcurrentLinkedQueue<Long> aclChangeSetsToReindex = new ConcurrentLinkedQueue<Long>();
    private ConcurrentLinkedQueue<Long> aclChangeSetsToIndex = new ConcurrentLinkedQueue<Long>();
    private ConcurrentLinkedQueue<Long> aclChangeSetsToPurge = new ConcurrentLinkedQueue<Long>();
@@ -159,7 +157,7 @@ public class AclTracker extends AbstractTracker
                //System.out.println("############## Indexing ACL ID:"+aclId);
                Acl acl = new Acl(0, aclId);
                List<AclReaders> readers = client.getAclReaders(Collections.singletonList(acl));
-                AclReaders r = readers.get(0);
+                //AclReaders r = readers.get(0);
                //System.out.println("############## READERS ID:"+r.getId()+":"+r.getReaders());
                indexAcl(readers, false);
            }
@@ -39,6 +39,7 @@ import org.slf4j.LoggerFactory;
 */
 public class SolrTrackerScheduler
 {
+    private static final String DEFAULT_CRON = "0/15 * * * * ? *";
    protected static final String SOLR_JOB_GROUP = "Solr";
    protected final static Logger log = LoggerFactory.getLogger(SolrTrackerScheduler.class);
    protected Scheduler scheduler;
@@ -83,7 +84,19 @@ public class SolrTrackerScheduler
        Trigger trigger;
        try
        {
-            String cron =  props.getProperty("alfresco.cron", "0/15 * * * * ? *");
+            String cron =  props.getProperty("alfresco.cron", DEFAULT_CRON);
+            if(tracker instanceof AclTracker) 
+            {
+                cron = props.getProperty("alfresco.tracker.acl.cron", DEFAULT_CRON);
+            }
+            if(tracker instanceof ContentTracker) 
+            {
+                cron = props.getProperty("alfresco.tracker.content.cron", DEFAULT_CRON);
+            }
+            if(tracker instanceof MetadataTracker) 
+            {
+                cron = props.getProperty("alfresco.tracker.metadata.cron", DEFAULT_CRON);
+            }
            trigger = new CronTrigger(jobName, SOLR_JOB_GROUP, cron);
            log.info("Scheduling job " + jobName);
            scheduler.scheduleJob(job, trigger);
@@ -93,7 +106,7 @@ public class SolrTrackerScheduler
            logError("Tracker", e);
        }
        catch (SchedulerException e)
-        {
+        {   
            logError("Tracker", e);
        }
    }
@@ -18,10 +18,11 @@
 */
 package org.alfresco.solr.tracker;

-import org.alfresco.solr.TrackerState;
-
+import java.util.Properties;
 import java.util.concurrent.Semaphore;

+import org.alfresco.solr.TrackerState;
+
 public interface Tracker
 {
    void track();
@@ -35,9 +36,13 @@ public interface Tracker
    String getAlfrescoVersion();
    
    void setShutdown(boolean shutdown);
+    
    void shutdown();

    boolean getRollback();
+    
+    Properties getProps();
+    
    void setRollback(boolean rollback);

    void invalidateState();
@@ -51,7 +51,7 @@ alfresco.encryption.ssl.truststore.provider=
 alfresco.encryption.ssl.truststore.location=ssl.repo.client.truststore
 alfresco.encryption.ssl.truststore.passwordFileLocation=ssl-truststore-passwords.properties

-# Tracking
+# Tracking //Deprecate this and push it down to tracker

 alfresco.corePoolSize=8
 alfresco.maximumPoolSize=-1
@@ -60,6 +60,11 @@ alfresco.threadPriority=5
 alfresco.threadDaemon=true
 alfresco.workQueueSize=-1

+# Breaking trackers
+alfresco.tracker.acl.cron="0/10 * * * * ? *"
+alfresco.tracker.content.cron="0/10 * * * * ? *"
+alfresco.tracker.metadata.cron="0/10 * * * * ? *"
+
 alfresco.commitInterval=2000
 alfresco.newSearcherInterval=3000

@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.math.BigInteger;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
@@ -39,6 +40,7 @@ import org.alfresco.solr.client.Node;
 import org.alfresco.solr.client.NodeMetaData;
 import org.alfresco.solr.client.SOLRAPIQueueClient;
 import org.alfresco.solr.client.Transaction;
+import org.alfresco.solr.tracker.Tracker;
 import org.apache.chemistry.opencmis.commons.impl.json.JSONArray;
 import org.apache.chemistry.opencmis.commons.impl.json.JSONObject;
 import org.apache.chemistry.opencmis.commons.impl.json.JSONValue;
@@ -76,6 +78,7 @@ import org.apache.solr.util.TestHarness;
 import org.apache.solr.util.TestHarness.TestCoresLocator;
 import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.xml.sax.SAXException;

 /**
@@ -89,6 +92,7 @@ import org.xml.sax.SAXException;
 */
 public abstract class  AbstractAlfrescoSolrTests implements SolrTestFiles, AlfrecsoSolrConstants
 {
+    static AlfrescoCoreAdminHandler admin;
    private static Log log = LogFactory.getLog(AbstractAlfrescoSolrTests.class);

    protected static SolrConfig solrConfig;
@@ -140,6 +144,7 @@ public abstract class  AbstractAlfrescoSolrTests implements SolrTestFiles, Alfre
            createAlfrescoCore(schema);
        }
        log.info("####initCore end");
+        admin = (AlfrescoCoreAdminHandler)h.getCore().getCoreContainer().getMultiCoreHandler();
    }

    public static void createAlfrescoCore(String schema) throws ParserConfigurationException, IOException, SAXException
@@ -149,6 +154,9 @@ public abstract class  AbstractAlfrescoSolrTests implements SolrTestFiles, Alfre
        properties.put("solr.tests.maxIndexingThreads", "10");
        properties.put("solr.tests.ramBufferSizeMB", "1024");
        properties.put("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler");
+        properties.put("alfresco.tracker.acl.cron", "0/10 * * * * ? *");
+        properties.put("alfresco.tracker.content.cron", "0/10 * * * * ? *");
+        properties.put("alfresco.tracker.metadata.cron", "0/10 * * * * ? *");
        if("schema.xml".equalsIgnoreCase(schema))
        {
            //currently this is hard coded to use the rerank production schema.
@@ -184,7 +192,7 @@ public abstract class  AbstractAlfrescoSolrTests implements SolrTestFiles, Alfre
        lrf = h.getRequestFactory
                ("standard",0,20, CommonParams.VERSION,"2.2");
    }
-    @AfterClass
+    @AfterClass()
    public static void tearDown()
    {
        solrConfig = null;
@@ -677,4 +685,9 @@ public abstract class  AbstractAlfrescoSolrTests implements SolrTestFiles, Alfre
            super(core, new MultiMapSolrParams(Collections.<String, String[]> emptyMap()));
        }
    }
+    protected Collection<Tracker> getTrackers() {
+        Collection<Tracker> trackers = admin.getTrackerRegistry().getTrackersForCore(h.getCore().getName());
+        log.info("######### Number of trackers is "+trackers.size()+" ###########");
+        return trackers;
+    }
 }
@@ -40,8 +40,6 @@ import org.quartz.SchedulerException;
 public class AlfrescoSolrReloadTest extends AbstractAlfrescoSolrTests {
    private static Log logger = LogFactory.getLog(org.alfresco.solr.tracker.AlfrescoSolrTrackerTest.class);

-    static AlfrescoCoreAdminHandler admin;
-
    @BeforeClass
    public static void beforeClass() throws Exception {
        initAlfrescoCore("schema.xml");
@@ -91,6 +89,8 @@ public class AlfrescoSolrReloadTest extends AbstractAlfrescoSolrTests {
        waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 1000, 100000);

        Collection<Tracker> trackers = getTrackers();
+        
+
        int numOfTrackers = trackers.size();
        int jobs = getJobsCount();

@@ -132,10 +132,4 @@ public class AlfrescoSolrReloadTest extends AbstractAlfrescoSolrTests {
        logger.info("######### Number of jobs is "+count+" ###########");
        return count;
    }
-
-    private Collection<Tracker> getTrackers() {
-        Collection<Tracker> trackers = admin.getTrackerRegistry().getTrackersForCore(h.getCore().getName());
-        logger.info("######### Number of trackers is "+trackers.size()+" ###########");
-        return trackers;
-    }
 }
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2005-2017 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.solr;
+
+import java.util.Collection;
+import java.util.Properties;
+
+import org.alfresco.solr.client.SOLRAPIQueueClient;
+import org.alfresco.solr.tracker.Tracker;
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+@SolrTestCaseJ4.SuppressSSL
+public class AlfrescoTrackerRegistrationTest extends AbstractAlfrescoSolrTests
+{
+    @BeforeClass
+    public static void beforeClass() throws Exception {
+        initAlfrescoCore("schema.xml");
+    }
+
+    @After
+    public void clearQueue() throws Exception {
+        SOLRAPIQueueClient.nodeMetaDataMap.clear();
+        SOLRAPIQueueClient.transactionQueue.clear();
+        SOLRAPIQueueClient.aclChangeSetQueue.clear();
+        SOLRAPIQueueClient.aclReadersMap.clear();
+        SOLRAPIQueueClient.aclMap.clear();
+        SOLRAPIQueueClient.nodeMap.clear();
+    }
+    @Test
+    public void checkCronOnTrackers()
+    {
+        Collection<Tracker> trackers = getTrackers();
+        Assert.assertNotNull(trackers);
+        trackers.forEach((tracker-> checkCronOnTracker(tracker)));
+    }
+    private void checkCronOnTracker(Tracker tracker)
+    {
+        Properties props = tracker.getProps();
+        Assert.assertEquals("0/10 * * * * ? *", props.get("alfresco.tracker.acl.cron"));
+        Assert.assertEquals("0/10 * * * * ? *", props.get("alfresco.tracker.content.cron"));
+        Assert.assertEquals("0/10 * * * * ? *", props.get("alfresco.tracker.metadata.cron"));
+    }
+}
@@ -22,6 +22,7 @@ package org.alfresco.solr.highlight;
 import org.alfresco.model.ContentModel;
 import org.alfresco.repo.search.adaptor.lucene.QueryConstants;
 import org.alfresco.solr.AbstractAlfrescoSolrTests;
+import org.alfresco.solr.AlfrescoCoreAdminHandler;
 import org.alfresco.solr.client.*;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -150,7 +151,7 @@ public class AlfrescoHighlighterTest extends AbstractAlfrescoSolrTests
        logger.info("######### Waiting for Doc Count ###########");

        waitForDocCount(new TermQuery(new Term(QueryConstants.FIELD_READER, "jim")), 1, MAX_WAIT_TIME);
-        waitForDocCount(new TermQuery(new Term(QueryConstants.FIELD_OWNER, "mike")), 4, 10000);
+        waitForDocCount(new TermQuery(new Term(QueryConstants.FIELD_OWNER, "mike")), 4, MAX_WAIT_TIME);

        logger.info("######### Testing SNIPPETS / FRAGSIZE ###########");
        SolrServletRequest req = areq(params( "q", "name:long", "qt", "/afts", "start", "0", "rows", "5",
@@ -80,14 +80,37 @@ public class SolrTrackerSchedulerTest
    }

    @Test
-    public void testSchedule() throws SchedulerException
+    public void testAclSchedule() throws SchedulerException
    {
        Properties props = mock(Properties.class);
-        when(props.getProperty("alfresco.cron", "0/15 * * * * ? *")).thenReturn("0/15 * * * * ? *");
+        when(props.getProperty("alfresco.tracker.acl.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *");
        this.trackerScheduler.schedule(aclTracker, CORE_NAME, props);
        verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class));
    }
-
+    @Test
+    public void testContentTrackerSchedule() throws SchedulerException
+    {
+        Properties props = mock(Properties.class);
+        when(props.getProperty("alfresco.tracker.content.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *");
+        this.trackerScheduler.schedule(contentTracker, CORE_NAME, props);
+        verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class));
+    }
+    @Test
+    public void testMetaDataTrackerSchedule() throws SchedulerException
+    {
+        Properties props = mock(Properties.class);
+        when(props.getProperty("alfresco.tracker.metadata.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *");
+        this.trackerScheduler.schedule(metadataTracker, CORE_NAME, props);
+        verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class));
+    }
+    @Test
+    public void testSchedule() throws SchedulerException
+    {
+        Properties props = mock(Properties.class);
+        when(props.getProperty("alfresco.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *");
+        this.trackerScheduler.schedule(cascadeTracker, CORE_NAME, props);
+        verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class));
+    }
    @Test
    public void testShutdown() throws SchedulerException
    {