From e83ab4d73aa65d847e723506c0494be6f8ec0736 Mon Sep 17 00:00:00 2001 From: Michael Suzuki Date: Thu, 22 Jun 2017 12:20:28 +0100 Subject: [PATCH 1/7] Search-77, added ability to configure tracker schedule to content, metadata and acl --- .../solr/lifecycle/SolrCoreLoadListener.java | 9 +-- .../lifecycle/SolrCoreLoadRegistration.java | 37 +++++++---- .../solr/tracker/AbstractTracker.java | 5 ++ .../org/alfresco/solr/tracker/AclTracker.java | 4 +- .../solr/tracker/SolrTrackerScheduler.java | 17 ++++- .../org/alfresco/solr/tracker/Tracker.java | 9 ++- .../templates/rerank/conf/solrcore.properties | 7 ++- .../solr/AbstractAlfrescoSolrTests.java | 15 ++++- .../alfresco/solr/AlfrescoSolrReloadTest.java | 10 +-- .../solr/AlfrescoTrackerRegistrationTest.java | 63 +++++++++++++++++++ .../highlight/AlfrescoHighlighterTest.java | 3 +- .../tracker/SolrTrackerSchedulerTest.java | 29 ++++++++- 12 files changed, 170 insertions(+), 38 deletions(-) create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationTest.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java index 219de41af..5f4e6db5f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java @@ -20,13 +20,10 @@ package org.alfresco.solr.lifecycle; import org.alfresco.solr.AlfrescoCoreAdminHandler; -import org.apache.solr.core.*; -import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.core.AbstractSolrEventListener; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; import org.apache.solr.search.SolrIndexSearcher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.invoke.MethodHandles; /** * Listens for the first searcher to be created for a core and registers the trackers diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java index 8febd10d9..d9e451799 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java @@ -18,22 +18,37 @@ */ package org.alfresco.solr.lifecycle; -import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService; -import org.alfresco.solr.*; -import org.alfresco.solr.client.SOLRAPIClient; -import org.alfresco.solr.client.SOLRAPIClientFactory; -import org.alfresco.solr.content.SolrContentStore; -import org.alfresco.solr.tracker.*; -import org.apache.solr.core.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Properties; +import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService; +import org.alfresco.solr.AlfrescoCoreAdminHandler; +import org.alfresco.solr.AlfrescoSolrDataModel; +import org.alfresco.solr.SolrInformationServer; +import org.alfresco.solr.SolrKeyResourceLoader; +import org.alfresco.solr.client.SOLRAPIClient; +import org.alfresco.solr.client.SOLRAPIClientFactory; +import org.alfresco.solr.content.SolrContentStore; +import org.alfresco.solr.tracker.AclTracker; +import org.alfresco.solr.tracker.CascadeTracker; +import org.alfresco.solr.tracker.CommitTracker; +import org.alfresco.solr.tracker.ContentTracker; +import org.alfresco.solr.tracker.MetadataTracker; +import org.alfresco.solr.tracker.ModelTracker; +import org.alfresco.solr.tracker.SolrTrackerScheduler; +import org.alfresco.solr.tracker.Tracker; +import org.alfresco.solr.tracker.TrackerRegistry; +import org.apache.solr.core.CloseHook; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.CoreDescriptorDecorator; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Deals with core registration when the core is loaded. * @@ -136,7 +151,7 @@ public class SolrCoreLoadRegistration { * @return A list of trackers */ private static List createTrackers(String coreName, TrackerRegistry trackerRegistry, Properties props, SolrTrackerScheduler scheduler, SOLRAPIClient repositoryClient, SolrInformationServer srv) { - List trackers = new ArrayList(); + List trackers = new ArrayList(); AclTracker aclTracker = new AclTracker(props, repositoryClient, coreName, srv); trackerRegistry.register(coreName, aclTracker); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java index da0573bcd..f45e2b13c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java @@ -305,6 +305,11 @@ public abstract class AbstractTracker implements Tracker { return alfrescoVersion; } + + public Properties getProps() + { + return props; + } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java index 32404ed22..1f2118be3 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java @@ -62,8 +62,6 @@ public class AclTracker extends AbstractTracker private int changeSetAclsBatchSize = DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE; private int aclBatchSize = DEFAULT_ACL_BATCH_SIZE; - private int maxAclChangeSetDocumentIdCacheSize = 700000; - private ConcurrentLinkedQueue aclChangeSetsToReindex = new ConcurrentLinkedQueue(); private ConcurrentLinkedQueue aclChangeSetsToIndex = new ConcurrentLinkedQueue(); private ConcurrentLinkedQueue aclChangeSetsToPurge = new ConcurrentLinkedQueue(); @@ -159,7 +157,7 @@ public class AclTracker extends AbstractTracker //System.out.println("############## Indexing ACL ID:"+aclId); Acl acl = new Acl(0, aclId); List readers = client.getAclReaders(Collections.singletonList(acl)); - AclReaders r = readers.get(0); + //AclReaders r = readers.get(0); //System.out.println("############## READERS ID:"+r.getId()+":"+r.getReaders()); indexAcl(readers, false); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java index 6db2c909e..39ed6f204 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java @@ -39,6 +39,7 @@ import org.slf4j.LoggerFactory; */ public class SolrTrackerScheduler { + private static final String DEFAULT_CRON = "0/15 * * * * ? *"; protected static final String SOLR_JOB_GROUP = "Solr"; protected final static Logger log = LoggerFactory.getLogger(SolrTrackerScheduler.class); protected Scheduler scheduler; @@ -83,7 +84,19 @@ public class SolrTrackerScheduler Trigger trigger; try { - String cron = props.getProperty("alfresco.cron", "0/15 * * * * ? *"); + String cron = props.getProperty("alfresco.cron", DEFAULT_CRON); + if(tracker instanceof AclTracker) + { + cron = props.getProperty("alfresco.tracker.acl.cron", DEFAULT_CRON); + } + if(tracker instanceof ContentTracker) + { + cron = props.getProperty("alfresco.tracker.content.cron", DEFAULT_CRON); + } + if(tracker instanceof MetadataTracker) + { + cron = props.getProperty("alfresco.tracker.metadata.cron", DEFAULT_CRON); + } trigger = new CronTrigger(jobName, SOLR_JOB_GROUP, cron); log.info("Scheduling job " + jobName); scheduler.scheduleJob(job, trigger); @@ -93,7 +106,7 @@ public class SolrTrackerScheduler logError("Tracker", e); } catch (SchedulerException e) - { + { logError("Tracker", e); } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java index 2f6fa41ce..522d363d5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java @@ -18,10 +18,11 @@ */ package org.alfresco.solr.tracker; -import org.alfresco.solr.TrackerState; - +import java.util.Properties; import java.util.concurrent.Semaphore; +import org.alfresco.solr.TrackerState; + public interface Tracker { void track(); @@ -35,9 +36,13 @@ public interface Tracker String getAlfrescoVersion(); void setShutdown(boolean shutdown); + void shutdown(); boolean getRollback(); + + Properties getProps(); + void setRollback(boolean rollback); void invalidateState(); diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index c812079b4..df9386d76 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -51,7 +51,7 @@ alfresco.encryption.ssl.truststore.provider= alfresco.encryption.ssl.truststore.location=ssl.repo.client.truststore alfresco.encryption.ssl.truststore.passwordFileLocation=ssl-truststore-passwords.properties -# Tracking +# Tracking //Deprecate this and push it down to tracker alfresco.corePoolSize=8 alfresco.maximumPoolSize=-1 @@ -60,6 +60,11 @@ alfresco.threadPriority=5 alfresco.threadDaemon=true alfresco.workQueueSize=-1 +# Breaking trackers +alfresco.tracker.acl.cron="0/10 * * * * ? *" +alfresco.tracker.content.cron="0/10 * * * * ? *" +alfresco.tracker.metadata.cron="0/10 * * * * ? *" + alfresco.commitInterval=2000 alfresco.newSearcherInterval=3000 diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java index da5b8487a..6c5e299e6 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.math.BigInteger; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; @@ -39,6 +40,7 @@ import org.alfresco.solr.client.Node; import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.client.SOLRAPIQueueClient; import org.alfresco.solr.client.Transaction; +import org.alfresco.solr.tracker.Tracker; import org.apache.chemistry.opencmis.commons.impl.json.JSONArray; import org.apache.chemistry.opencmis.commons.impl.json.JSONObject; import org.apache.chemistry.opencmis.commons.impl.json.JSONValue; @@ -76,6 +78,7 @@ import org.apache.solr.util.TestHarness; import org.apache.solr.util.TestHarness.TestCoresLocator; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.xml.sax.SAXException; /** @@ -89,6 +92,7 @@ import org.xml.sax.SAXException; */ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, AlfrecsoSolrConstants { + static AlfrescoCoreAdminHandler admin; private static Log log = LogFactory.getLog(AbstractAlfrescoSolrTests.class); protected static SolrConfig solrConfig; @@ -140,6 +144,7 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfre createAlfrescoCore(schema); } log.info("####initCore end"); + admin = (AlfrescoCoreAdminHandler)h.getCore().getCoreContainer().getMultiCoreHandler(); } public static void createAlfrescoCore(String schema) throws ParserConfigurationException, IOException, SAXException @@ -149,6 +154,9 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfre properties.put("solr.tests.maxIndexingThreads", "10"); properties.put("solr.tests.ramBufferSizeMB", "1024"); properties.put("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler"); + properties.put("alfresco.tracker.acl.cron", "0/10 * * * * ? *"); + properties.put("alfresco.tracker.content.cron", "0/10 * * * * ? *"); + properties.put("alfresco.tracker.metadata.cron", "0/10 * * * * ? *"); if("schema.xml".equalsIgnoreCase(schema)) { //currently this is hard coded to use the rerank production schema. @@ -184,7 +192,7 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfre lrf = h.getRequestFactory ("standard",0,20, CommonParams.VERSION,"2.2"); } - @AfterClass + @AfterClass() public static void tearDown() { solrConfig = null; @@ -677,4 +685,9 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfre super(core, new MultiMapSolrParams(Collections. emptyMap())); } } + protected Collection getTrackers() { + Collection trackers = admin.getTrackerRegistry().getTrackersForCore(h.getCore().getName()); + log.info("######### Number of trackers is "+trackers.size()+" ###########"); + return trackers; + } } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadTest.java index 132914648..f49944df7 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadTest.java @@ -40,8 +40,6 @@ import org.quartz.SchedulerException; public class AlfrescoSolrReloadTest extends AbstractAlfrescoSolrTests { private static Log logger = LogFactory.getLog(org.alfresco.solr.tracker.AlfrescoSolrTrackerTest.class); - static AlfrescoCoreAdminHandler admin; - @BeforeClass public static void beforeClass() throws Exception { initAlfrescoCore("schema.xml"); @@ -91,6 +89,8 @@ public class AlfrescoSolrReloadTest extends AbstractAlfrescoSolrTests { waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 1000, 100000); Collection trackers = getTrackers(); + + int numOfTrackers = trackers.size(); int jobs = getJobsCount(); @@ -132,10 +132,4 @@ public class AlfrescoSolrReloadTest extends AbstractAlfrescoSolrTests { logger.info("######### Number of jobs is "+count+" ###########"); return count; } - - private Collection getTrackers() { - Collection trackers = admin.getTrackerRegistry().getTrackersForCore(h.getCore().getName()); - logger.info("######### Number of trackers is "+trackers.size()+" ###########"); - return trackers; - } } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationTest.java new file mode 100644 index 000000000..cbb1c1e36 --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationTest.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2005-2017 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr; + +import java.util.Collection; +import java.util.Properties; + +import org.alfresco.solr.client.SOLRAPIQueueClient; +import org.alfresco.solr.tracker.Tracker; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.After; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +@SolrTestCaseJ4.SuppressSSL +public class AlfrescoTrackerRegistrationTest extends AbstractAlfrescoSolrTests +{ + @BeforeClass + public static void beforeClass() throws Exception { + initAlfrescoCore("schema.xml"); + } + + @After + public void clearQueue() throws Exception { + SOLRAPIQueueClient.nodeMetaDataMap.clear(); + SOLRAPIQueueClient.transactionQueue.clear(); + SOLRAPIQueueClient.aclChangeSetQueue.clear(); + SOLRAPIQueueClient.aclReadersMap.clear(); + SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.nodeMap.clear(); + } + @Test + public void checkCronOnTrackers() + { + Collection trackers = getTrackers(); + Assert.assertNotNull(trackers); + trackers.forEach((tracker-> checkCronOnTracker(tracker))); + } + private void checkCronOnTracker(Tracker tracker) + { + Properties props = tracker.getProps(); + Assert.assertEquals("0/10 * * * * ? *", props.get("alfresco.tracker.acl.cron")); + Assert.assertEquals("0/10 * * * * ? *", props.get("alfresco.tracker.content.cron")); + Assert.assertEquals("0/10 * * * * ? *", props.get("alfresco.tracker.metadata.cron")); + } +} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterTest.java index e28ab9645..a3639cb6f 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterTest.java @@ -22,6 +22,7 @@ package org.alfresco.solr.highlight; import org.alfresco.model.ContentModel; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.alfresco.solr.client.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -150,7 +151,7 @@ public class AlfrescoHighlighterTest extends AbstractAlfrescoSolrTests logger.info("######### Waiting for Doc Count ###########"); waitForDocCount(new TermQuery(new Term(QueryConstants.FIELD_READER, "jim")), 1, MAX_WAIT_TIME); - waitForDocCount(new TermQuery(new Term(QueryConstants.FIELD_OWNER, "mike")), 4, 10000); + waitForDocCount(new TermQuery(new Term(QueryConstants.FIELD_OWNER, "mike")), 4, MAX_WAIT_TIME); logger.info("######### Testing SNIPPETS / FRAGSIZE ###########"); SolrServletRequest req = areq(params( "q", "name:long", "qt", "/afts", "start", "0", "rows", "5", diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java index 79d52ec90..d005675b2 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java @@ -80,14 +80,37 @@ public class SolrTrackerSchedulerTest } @Test - public void testSchedule() throws SchedulerException + public void testAclSchedule() throws SchedulerException { Properties props = mock(Properties.class); - when(props.getProperty("alfresco.cron", "0/15 * * * * ? *")).thenReturn("0/15 * * * * ? *"); + when(props.getProperty("alfresco.tracker.acl.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *"); this.trackerScheduler.schedule(aclTracker, CORE_NAME, props); verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); } - + @Test + public void testContentTrackerSchedule() throws SchedulerException + { + Properties props = mock(Properties.class); + when(props.getProperty("alfresco.tracker.content.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *"); + this.trackerScheduler.schedule(contentTracker, CORE_NAME, props); + verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); + } + @Test + public void testMetaDataTrackerSchedule() throws SchedulerException + { + Properties props = mock(Properties.class); + when(props.getProperty("alfresco.tracker.metadata.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *"); + this.trackerScheduler.schedule(metadataTracker, CORE_NAME, props); + verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); + } + @Test + public void testSchedule() throws SchedulerException + { + Properties props = mock(Properties.class); + when(props.getProperty("alfresco.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *"); + this.trackerScheduler.schedule(cascadeTracker, CORE_NAME, props); + verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); + } @Test public void testShutdown() throws SchedulerException { From 4d918ff05081ad7ef746d3d0471ec29c05306d32 Mon Sep 17 00:00:00 2001 From: Michael Suzuki Date: Thu, 22 Jun 2017 15:47:53 +0100 Subject: [PATCH 2/7] Search-77, amend proerty naming pattern and added commit and cascade tracker --- .../solr/tracker/SolrTrackerScheduler.java | 19 +++++++--- .../templates/rerank/conf/solrcore.properties | 12 +++---- .../solr/AbstractAlfrescoSolrTests.java | 8 +++-- .../tracker/SolrTrackerSchedulerTest.java | 35 ++++++++++++++++--- 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java index 39ed6f204..016e3b422 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java @@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory; public class SolrTrackerScheduler { private static final String DEFAULT_CRON = "0/15 * * * * ? *"; - protected static final String SOLR_JOB_GROUP = "Solr"; + public static final String SOLR_JOB_GROUP = "Solr"; protected final static Logger log = LoggerFactory.getLogger(SolrTrackerScheduler.class); protected Scheduler scheduler; @@ -84,19 +84,28 @@ public class SolrTrackerScheduler Trigger trigger; try { - String cron = props.getProperty("alfresco.cron", DEFAULT_CRON); + String cron = null; if(tracker instanceof AclTracker) { - cron = props.getProperty("alfresco.tracker.acl.cron", DEFAULT_CRON); + cron = props.getProperty("alfresco.acl.tracker.cron", DEFAULT_CRON); } if(tracker instanceof ContentTracker) { - cron = props.getProperty("alfresco.tracker.content.cron", DEFAULT_CRON); + cron = props.getProperty("alfresco.content.tracker.cron", DEFAULT_CRON); } if(tracker instanceof MetadataTracker) { - cron = props.getProperty("alfresco.tracker.metadata.cron", DEFAULT_CRON); + cron = props.getProperty("alfresco.metadata.tracker.cron", DEFAULT_CRON); } + if(tracker instanceof CascadeTracker) + { + cron = props.getProperty("alfresco.cascade.tracker.cron", DEFAULT_CRON); + } + if(tracker instanceof CommitTracker) + { + cron = props.getProperty("alfresco.commit.tracker.cron", DEFAULT_CRON); + } + cron = cron == null ? DEFAULT_CRON : cron; trigger = new CronTrigger(jobName, SOLR_JOB_GROUP, cron); log.info("Scheduling job " + jobName); scheduler.scheduleJob(job, trigger); diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index df9386d76..5217a9205 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -24,7 +24,6 @@ alfresco.host=localhost alfresco.port=8080 alfresco.port.ssl=8443 alfresco.baseUrl=/alfresco -alfresco.cron=0/10 * * * * ? * #alfresco.index.transformContent=false #alfresco.ignore.datatype.1=d:content @@ -59,11 +58,12 @@ alfresco.keepAliveTime=120 alfresco.threadPriority=5 alfresco.threadDaemon=true alfresco.workQueueSize=-1 - -# Breaking trackers -alfresco.tracker.acl.cron="0/10 * * * * ? *" -alfresco.tracker.content.cron="0/10 * * * * ? *" -alfresco.tracker.metadata.cron="0/10 * * * * ? *" +#Track configuration +alfresco.acl.tracker.cron=0/15 * * * * ? * +alfresco.content.tracker.cron=0/15 * * * * ? * +alfresco.metadata.tracker.cron=0/15 * * * * ? * +alfresco.cascade.tracker.cron=0/15 * * * * ? * +alfresco.commit.tracker.cron=0/15 * * * * ? * alfresco.commitInterval=2000 alfresco.newSearcherInterval=3000 diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java index 6c5e299e6..18cef16e4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java @@ -154,9 +154,11 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfre properties.put("solr.tests.maxIndexingThreads", "10"); properties.put("solr.tests.ramBufferSizeMB", "1024"); properties.put("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler"); - properties.put("alfresco.tracker.acl.cron", "0/10 * * * * ? *"); - properties.put("alfresco.tracker.content.cron", "0/10 * * * * ? *"); - properties.put("alfresco.tracker.metadata.cron", "0/10 * * * * ? *"); + properties.put("alfresco.acl.tracker.cron", "0/10 * * * * ? *"); + properties.put("alfresco.content.tracker.cron", "0/10 * * * * ? *"); + properties.put("alfresco.metadata.tracker.cron", "0/10 * * * * ? *"); + properties.put("alfresco.cascade.tracker.cron", "0/10 * * * * ? *"); + properties.put("alfresco.commit.tracker.cron", "0/10 * * * * ? *"); if("schema.xml".equalsIgnoreCase(schema)) { //currently this is hard coded to use the rerank production schema. diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java index d005675b2..b28134c3c 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java @@ -26,20 +26,25 @@ import static org.mockito.Matchers.eq; import static org.mockito.Mockito.*; import java.util.Arrays; +import java.util.Date; +import java.util.List; import java.util.Properties; import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; +import org.quartz.CronTrigger; import org.quartz.JobDetail; import org.quartz.Scheduler; import org.quartz.SchedulerException; import org.quartz.Trigger; + @RunWith(MockitoJUnitRunner.class) public class SolrTrackerSchedulerTest { @@ -82,34 +87,54 @@ public class SolrTrackerSchedulerTest @Test public void testAclSchedule() throws SchedulerException { + String exp = "0/12 * * * * ? *"; Properties props = mock(Properties.class); - when(props.getProperty("alfresco.tracker.acl.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *"); + when(props.getProperty("alfresco.acl.tracker.cron", "0/15 * * * * ? *")).thenReturn(exp); this.trackerScheduler.schedule(aclTracker, CORE_NAME, props); verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); + checkCronExpression(exp); } @Test public void testContentTrackerSchedule() throws SchedulerException { Properties props = mock(Properties.class); - when(props.getProperty("alfresco.tracker.content.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *"); + String exp = "0/5 * * * * ? *"; + when(props.getProperty("alfresco.content.tracker.cron", "0/15 * * * * ? *")).thenReturn(exp); this.trackerScheduler.schedule(contentTracker, CORE_NAME, props); verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); + //loop all jobs by groupname + checkCronExpression(exp); + + } + private void checkCronExpression(String exp) throws SchedulerException + { + for (String jobName : this.trackerScheduler.scheduler.getJobNames(SolrTrackerScheduler.SOLR_JOB_GROUP)) + { + Trigger[] triggers = this.trackerScheduler.scheduler.getTriggersOfJob(jobName,SolrTrackerScheduler.SOLR_JOB_GROUP); + CronTrigger t = (CronTrigger) triggers[0]; + String cronExp = t.getCronExpression(); + Assert.assertEquals(exp, cronExp); + } } @Test public void testMetaDataTrackerSchedule() throws SchedulerException { + String exp = "0/4 * * * * ? *"; Properties props = mock(Properties.class); - when(props.getProperty("alfresco.tracker.metadata.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *"); + when(props.getProperty("alfresco.metadata.tracker.cron", "0/15 * * * * ? *")).thenReturn(exp); this.trackerScheduler.schedule(metadataTracker, CORE_NAME, props); verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); + checkCronExpression(exp); } @Test public void testSchedule() throws SchedulerException { + String exp = "0/1 * * * * ? *"; Properties props = mock(Properties.class); - when(props.getProperty("alfresco.cron", "0/15 * * * * ? *")).thenReturn("0/10 * * * * ? *"); + when(props.getProperty("alfresco.cascade.tracker.cron", "0/15 * * * * ? *")).thenReturn(exp); this.trackerScheduler.schedule(cascadeTracker, CORE_NAME, props); verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); + checkCronExpression(exp); } @Test public void testShutdown() throws SchedulerException @@ -137,7 +162,7 @@ public class SolrTrackerSchedulerTest public void testDeleteTrackerInstanceJob() throws SchedulerException { Properties props = mock(Properties.class); - when(props.getProperty("alfresco.cron", "0/15 * * * * ? *")).thenReturn("0/15 * * * * ? *"); + when(props.getProperty("alfresco.cascade.tracker.cron", "0/15 * * * * ? *")).thenReturn("0/15 * * * * ? *"); this.trackerScheduler.schedule(cascadeTrackerScheduled, CORE_NAME, props); //Try deleting the same class but a different instance. It not possible. From 17fc20467edbe9b2b30cff0980f23f37377cae34 Mon Sep 17 00:00:00 2001 From: Michael Suzuki Date: Fri, 23 Jun 2017 10:23:42 +0100 Subject: [PATCH 3/7] Tidy imports --- .../alfresco/solr/tracker/SolrTrackerSchedulerTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java index b28134c3c..7b5d92bd4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java @@ -23,11 +23,14 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Arrays; -import java.util.Date; -import java.util.List; import java.util.Properties; import org.alfresco.solr.AlfrescoCoreAdminHandler; From 327fefde27a16f70c4b69c530848801376460a83 Mon Sep 17 00:00:00 2001 From: Michael Suzuki Date: Fri, 23 Jun 2017 15:49:28 +0100 Subject: [PATCH 4/7] Search-77, refactor test to use swith and hardend the tests --- .../solr/tracker/AbstractTracker.java | 14 +++- .../org/alfresco/solr/tracker/AclTracker.java | 4 +- .../alfresco/solr/tracker/CascadeTracker.java | 26 +++---- .../alfresco/solr/tracker/CommitTracker.java | 6 +- .../alfresco/solr/tracker/ContentTracker.java | 7 +- .../solr/tracker/MetadataTracker.java | 12 ++- .../alfresco/solr/tracker/ModelTracker.java | 9 +-- .../solr/tracker/SolrTrackerScheduler.java | 39 ++++++---- .../org/alfresco/solr/tracker/Tracker.java | 11 +++ .../templates/rerank/conf/solrcore.properties | 2 +- .../solr/tracker/ContentTrackerTest.java | 7 ++ .../tracker/SolrTrackerSchedulerTest.java | 78 ++++++++++++------- 12 files changed, 131 insertions(+), 84 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java index f45e2b13c..e08a9d7b3 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java @@ -69,6 +69,7 @@ public abstract class AbstractTracker implements Tracker protected boolean transformContent; protected String shardTemplate; protected volatile boolean rollback; + protected final Type type; /* @@ -82,11 +83,12 @@ public abstract class AbstractTracker implements Tracker /** * Default constructor, strictly for testing. */ - protected AbstractTracker() + protected AbstractTracker(Type type) { + this.type = type; } - protected AbstractTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) + protected AbstractTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer,Type type) { this.props = p; this.client = client; @@ -110,6 +112,9 @@ public abstract class AbstractTracker implements Tracker this.trackerStats = this.infoSrv.getTrackerStats(); alfrescoVersion = p.getProperty("alfresco.version", "5.0.0"); + + this.type = type; + log.info("Solr built for Alfresco version: " + alfrescoVersion); } @@ -310,6 +315,11 @@ public abstract class AbstractTracker implements Tracker { return props; } + + public Type getType() + { + return type; + } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java index 1f2118be3..ac16758d1 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java @@ -75,13 +75,13 @@ public class AclTracker extends AbstractTracker */ AclTracker() { - super(); + super(Tracker.Type.ACL); } public AclTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) { - super(p, client, coreName, informationServer); + super(p, client, coreName, informationServer, Tracker.Type.ACL); changeSetAclsBatchSize = Integer.parseInt(p.getProperty("alfresco.changeSetAclsBatchSize", "100")); aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize", "10")); shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java index 3dac07080..ca151a5b1 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java @@ -19,22 +19,18 @@ package org.alfresco.solr.tracker; import java.io.IOException; -import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; +import java.util.Set; -import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.AuthenticationException; -import org.alfresco.repo.index.shard.ShardMethodEnum; -import org.alfresco.repo.index.shard.ShardState; -import org.alfresco.repo.index.shard.ShardStateBuilder; -import org.alfresco.solr.AlfrescoSolrDataModel; -import org.alfresco.solr.BoundedDeque; import org.alfresco.solr.InformationServer; -import org.alfresco.solr.NodeReport; -import org.alfresco.solr.TrackerState; -import org.alfresco.solr.adapters.IOpenBitSet; -import org.alfresco.solr.client.*; -import org.alfresco.solr.client.Node.SolrApiNodeStatus; +import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.SOLRAPIClient; +import org.alfresco.solr.client.Transaction; import org.apache.commons.codec.EncoderException; import org.json.JSONException; import org.slf4j.Logger; @@ -54,14 +50,14 @@ public class CascadeTracker extends AbstractTracker implements Tracker public CascadeTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) { - super(p, client, coreName, informationServer); + super(p, client, coreName, informationServer, Tracker.Type.Cascade); threadHandler = new ThreadHandler(p, coreName, "CascadeTracker"); } CascadeTracker() { - // Testing purposes only + super(Tracker.Type.Cascade); } @Override diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java index 185cb9ab7..b1569502b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java @@ -21,9 +21,7 @@ package org.alfresco.solr.tracker; import java.util.List; import java.util.Properties; -import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import org.alfresco.solr.InformationServer; import org.alfresco.solr.client.SOLRAPIClient; @@ -53,7 +51,7 @@ public class CommitTracker extends AbstractTracker **/ CommitTracker() { - super(); + super(Tracker.Type.Commit); } public CommitTracker(Properties p, @@ -62,7 +60,7 @@ public class CommitTracker extends AbstractTracker InformationServer informationServer, List trackers) { - super(p, client, coreName, informationServer); + super(p, client, coreName, informationServer, Tracker.Type.Commit); //Set the trackers for(Tracker tracker : trackers) { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java index eba54c27b..28b4fc698 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java @@ -21,9 +21,6 @@ import java.io.IOException; import java.util.List; import java.util.Properties; -import java.util.Set; -import java.util.WeakHashMap; -import java.util.concurrent.Semaphore; import org.alfresco.solr.AlfrescoSolrDataModel.TenantAclIdDbId; import org.alfresco.solr.InformationServer; @@ -48,7 +45,7 @@ public class ContentTracker extends AbstractTracker implements Tracker public ContentTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) { - super(p, client, coreName, informationServer); + super(p, client, coreName, informationServer, Tracker.Type.Content); contentReadBatchSize = Integer.parseInt(p.getProperty("alfresco.contentReadBatchSize", "100")); contentUpdateBatchSize = Integer.parseInt(p.getProperty("alfresco.contentUpdateBatchSize", "1000")); threadHandler = new ThreadHandler(p, coreName, "ContentTracker"); @@ -56,7 +53,7 @@ public class ContentTracker extends AbstractTracker implements Tracker ContentTracker() { - // Testing purposes only + super(Tracker.Type.Content); } @Override diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 3dc6ec74b..253bc2559 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -19,7 +19,12 @@ package org.alfresco.solr.tracker; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import org.alfresco.error.AlfrescoRuntimeException; @@ -62,7 +67,6 @@ public class MetadataTracker extends AbstractTracker implements Tracker private static final int DEFAULT_NODE_BATCH_SIZE = 10; private int transactionDocsBatchSize = DEFAULT_TRANSACTION_DOCS_BATCH_SIZE; private int nodeBatchSize = DEFAULT_NODE_BATCH_SIZE; - private int maxTransactionDocumentIdCacheSize = 700000; private ConcurrentLinkedQueue transactionsToReindex = new ConcurrentLinkedQueue(); private ConcurrentLinkedQueue transactionsToIndex = new ConcurrentLinkedQueue(); private ConcurrentLinkedQueue transactionsToPurge = new ConcurrentLinkedQueue(); @@ -76,7 +80,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker public MetadataTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) { - super(p, client, coreName, informationServer); + super(p, client, coreName, informationServer, Tracker.Type.MetaData); //System.out.println("####### MetadatTracker() ########"); transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); @@ -92,7 +96,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker MetadataTracker() { - // Testing purposes only + super(Tracker.Type.MetaData); } @Override diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java index 8d0b399ca..07b75c6d5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java @@ -32,11 +32,6 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; -import javax.naming.Context; -import javax.naming.InitialContext; -import javax.naming.NamingException; -import javax.naming.NoInitialContextException; - import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.AuthenticationException; import org.alfresco.repo.dictionary.M2Model; @@ -108,7 +103,7 @@ public class ModelTracker extends AbstractTracker implements Tracker public ModelTracker(String solrHome, Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) { - super(p, client, coreName, informationServer); + super(p, client, coreName, informationServer, Tracker.Type.Model); String normalSolrHome = SolrResourceLoader.normalizeDir(solrHome); alfrescoModelDir = new File(ConfigUtil.locateProperty("solr.model.dir", normalSolrHome+"alfrescoModels")); log.info("Alfresco Model dir " + alfrescoModelDir); @@ -192,7 +187,7 @@ public class ModelTracker extends AbstractTracker implements Tracker */ ModelTracker() { - // Testing purposes only + super(Tracker.Type.Model); } @Override diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java index 016e3b422..77801a3f5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java @@ -73,7 +73,14 @@ public class SolrTrackerScheduler { log.error("Failed to schedule " + jobType + " Job.", e); } - + /** + * Schedules individual trackers based on the solrcore properties. + * + * @author Michael Suzuki + * @param tracker + * @param coreName + * @param props + */ public void schedule(Tracker tracker, String coreName, Properties props) { String jobName = this.getJobName(tracker, coreName); @@ -85,27 +92,29 @@ public class SolrTrackerScheduler try { String cron = null; - if(tracker instanceof AclTracker) + switch (tracker.getType()) { + case ACL: cron = props.getProperty("alfresco.acl.tracker.cron", DEFAULT_CRON); - } - if(tracker instanceof ContentTracker) - { + break; + case Model: + cron = props.getProperty("alfresco.model.tracker.cron", DEFAULT_CRON); + break; + case Content: cron = props.getProperty("alfresco.content.tracker.cron", DEFAULT_CRON); - } - if(tracker instanceof MetadataTracker) - { + break; + case MetaData: cron = props.getProperty("alfresco.metadata.tracker.cron", DEFAULT_CRON); - } - if(tracker instanceof CascadeTracker) - { + break; + case Cascade: cron = props.getProperty("alfresco.cascade.tracker.cron", DEFAULT_CRON); - } - if(tracker instanceof CommitTracker) - { + break; + case Commit: cron = props.getProperty("alfresco.commit.tracker.cron", DEFAULT_CRON); + break; + default: props.getProperty(DEFAULT_CRON); + break; } - cron = cron == null ? DEFAULT_CRON : cron; trigger = new CronTrigger(jobName, SOLR_JOB_GROUP, cron); log.info("Scheduling job " + jobName); scheduler.scheduleJob(job, trigger); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java index 522d363d5..194bd1867 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java @@ -48,4 +48,15 @@ public interface Tracker void invalidateState(); TrackerState getTrackerState(); + + Type getType(); + + enum Type{ + Model, + Content, + ACL, + Cascade, + Commit, + MetaData + } } diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index 5217a9205..d77ded20e 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -64,7 +64,7 @@ alfresco.content.tracker.cron=0/15 * * * * ? * alfresco.metadata.tracker.cron=0/15 * * * * ? * alfresco.cascade.tracker.cron=0/15 * * * * ? * alfresco.commit.tracker.cron=0/15 * * * * ? * - +alfresco.model.tracker.cron=0/15 * * * * ? * alfresco.commitInterval=2000 alfresco.newSearcherInterval=3000 diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java index 8d51db4d2..a2e74acfb 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java @@ -27,6 +27,7 @@ import java.util.Properties; import org.alfresco.solr.AlfrescoSolrDataModel.TenantAclIdDbId; import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.SOLRAPIClient; +import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -133,4 +134,10 @@ public class ContentTrackerTest order.verify(srv).getDocsWithUncleanContent(0 + READ_BATCH + READ_BATCH, READ_BATCH); } + @Test + public void typeCheck() + { + Assert.assertTrue(contentTracker.getType().equals(Tracker.Type.Content)); + } + } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java index 7b5d92bd4..4a52146a6 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java @@ -33,7 +33,10 @@ import static org.mockito.Mockito.when; import java.util.Arrays; import java.util.Properties; +import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.solr.AlfrescoCoreAdminHandler; +import org.alfresco.solr.InformationServer; +import org.alfresco.solr.client.SOLRAPIClient; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -53,29 +56,33 @@ public class SolrTrackerSchedulerTest { @Mock private AlfrescoCoreAdminHandler adminHandler; - @Mock - private ModelTracker modelTracker; - @Mock - private ContentTracker contentTracker; - @Mock - private MetadataTracker metadataTracker; - @Mock - private AclTracker aclTracker; - @Mock - private CascadeTracker cascadeTracker; - @Mock - private CascadeTracker cascadeTrackerScheduled; - private SolrTrackerScheduler trackerScheduler; private String CORE_NAME = "coreName"; private Scheduler spiedQuartzScheduler; - + @Mock + SOLRAPIClient client; + String coreName = "alfresco"; + @Mock + InformationServer informationServer; + Properties props; @Before public void setUp() throws Exception { this.trackerScheduler = new SolrTrackerScheduler(adminHandler); this.spiedQuartzScheduler = spy(this.trackerScheduler.scheduler); this.trackerScheduler.scheduler = spiedQuartzScheduler; + props = new Properties(); + props.put("alfresco.stores", "workspace://SpacesStore"); + props.put("alfresco.batch.count", "5000"); + props.put("alfresco.maxLiveSearchers", "2"); + props.put("enable.slave", "false"); + props.put("enable.master", "true"); + props.put("shard.count", "1"); + props.put("shard.instance", "0"); + props.put("shard.method", "SHARD_METHOD_DBID"); + props.put("alfresco.template", ""); + props.put("alfresco.index.transformContent", "true"); + props.put("alfresco.version", "5.0.0"); } @After @@ -91,8 +98,8 @@ public class SolrTrackerSchedulerTest public void testAclSchedule() throws SchedulerException { String exp = "0/12 * * * * ? *"; - Properties props = mock(Properties.class); - when(props.getProperty("alfresco.acl.tracker.cron", "0/15 * * * * ? *")).thenReturn(exp); + props.put("alfresco.acl.tracker.cron", exp); + AclTracker aclTracker = new AclTracker(props, client, coreName, informationServer); this.trackerScheduler.schedule(aclTracker, CORE_NAME, props); verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); checkCronExpression(exp); @@ -100,15 +107,14 @@ public class SolrTrackerSchedulerTest @Test public void testContentTrackerSchedule() throws SchedulerException { - Properties props = mock(Properties.class); String exp = "0/5 * * * * ? *"; - when(props.getProperty("alfresco.content.tracker.cron", "0/15 * * * * ? *")).thenReturn(exp); + props.put("alfresco.content.tracker.cron", exp); + ContentTracker contentTracker = new ContentTracker(props, client, coreName, informationServer); this.trackerScheduler.schedule(contentTracker, CORE_NAME, props); verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); - //loop all jobs by groupname checkCronExpression(exp); - } + private void checkCronExpression(String exp) throws SchedulerException { for (String jobName : this.trackerScheduler.scheduler.getJobNames(SolrTrackerScheduler.SOLR_JOB_GROUP)) @@ -123,8 +129,8 @@ public class SolrTrackerSchedulerTest public void testMetaDataTrackerSchedule() throws SchedulerException { String exp = "0/4 * * * * ? *"; - Properties props = mock(Properties.class); - when(props.getProperty("alfresco.metadata.tracker.cron", "0/15 * * * * ? *")).thenReturn(exp); + props.put("alfresco.metadata.tracker.cron", exp); + MetadataTracker metadataTracker = new MetadataTracker(props, client, exp, informationServer); this.trackerScheduler.schedule(metadataTracker, CORE_NAME, props); verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); checkCronExpression(exp); @@ -133,8 +139,8 @@ public class SolrTrackerSchedulerTest public void testSchedule() throws SchedulerException { String exp = "0/1 * * * * ? *"; - Properties props = mock(Properties.class); - when(props.getProperty("alfresco.cascade.tracker.cron", "0/15 * * * * ? *")).thenReturn(exp); + props.put("alfresco.cascade.tracker.cron",exp); + CascadeTracker cascadeTracker = new CascadeTracker(props, client, exp, informationServer); this.trackerScheduler.schedule(cascadeTracker, CORE_NAME, props); verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); checkCronExpression(exp); @@ -149,6 +155,9 @@ public class SolrTrackerSchedulerTest @Test public void testDeleteTrackerJobs() throws SchedulerException { + ContentTracker contentTracker = new ContentTracker(); + MetadataTracker metadataTracker = new MetadataTracker(); + AclTracker aclTracker = new AclTracker(); this.trackerScheduler.deleteTrackerJobs(CORE_NAME, Arrays.asList(new Tracker[] { contentTracker, metadataTracker, aclTracker })); verify(spiedQuartzScheduler, times(3)).deleteJob(anyString(), eq(SolrTrackerScheduler.SOLR_JOB_GROUP)); @@ -157,6 +166,7 @@ public class SolrTrackerSchedulerTest @Test public void testDeleteTrackerJob() throws SchedulerException { + ModelTracker modelTracker = new ModelTracker(); this.trackerScheduler.deleteTrackerJob(CORE_NAME, modelTracker); verify(spiedQuartzScheduler).deleteJob(anyString(), eq(SolrTrackerScheduler.SOLR_JOB_GROUP)); } @@ -164,16 +174,16 @@ public class SolrTrackerSchedulerTest @Test public void testDeleteTrackerInstanceJob() throws SchedulerException { - Properties props = mock(Properties.class); - when(props.getProperty("alfresco.cascade.tracker.cron", "0/15 * * * * ? *")).thenReturn("0/15 * * * * ? *"); - this.trackerScheduler.schedule(cascadeTrackerScheduled, CORE_NAME, props); + CascadeTracker cascadeTracker = new CascadeTracker(props, client, coreName, informationServer); + CascadeTracker cascadeTracker2 = new CascadeTracker(props, client, coreName, informationServer); + this.trackerScheduler.schedule(cascadeTracker, CORE_NAME, props); //Try deleting the same class but a different instance. It not possible. - this.trackerScheduler.deleteJobForTrackerInstance(CORE_NAME, cascadeTracker); + this.trackerScheduler.deleteJobForTrackerInstance(CORE_NAME, cascadeTracker2); verify(spiedQuartzScheduler, never()).deleteJob(anyString(), eq(SolrTrackerScheduler.SOLR_JOB_GROUP)); //No try deleting the exact instance of the tracker class - this.trackerScheduler.deleteJobForTrackerInstance(CORE_NAME, cascadeTrackerScheduled); + this.trackerScheduler.deleteJobForTrackerInstance(CORE_NAME, cascadeTracker); verify(spiedQuartzScheduler, times(1)).deleteJob(anyString(), eq(SolrTrackerScheduler.SOLR_JOB_GROUP)); } @@ -190,5 +200,15 @@ public class SolrTrackerSchedulerTest this.trackerScheduler.pauseAll(); verify(this.spiedQuartzScheduler).pauseAll(); } + @Test + public void testModeltTrackerSchedule() throws SchedulerException + { + String exp = "0/20 * * * * ? *"; + props.put("alfresco.model.tracker.cron", exp); + ModelTracker modelTracker = new ModelTracker("alfresco", props, client, CORE_NAME, informationServer); + this.trackerScheduler.schedule(modelTracker, CORE_NAME, props); + verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); + checkCronExpression(exp); + } } From ebe3140b27739e15e9fe5b2113b8d3d02edb17f4 Mon Sep 17 00:00:00 2001 From: Michael Suzuki Date: Mon, 26 Jun 2017 09:43:39 +0100 Subject: [PATCH 5/7] Tidy code --- .../org/alfresco/solr/tracker/AclTracker.java | 2 +- .../alfresco/solr/tracker/CascadeTracker.java | 8 +++--- .../solr/tracker/MetadataTracker.java | 25 ------------------- .../templates/rerank/conf/solrcore.properties | 3 +-- 4 files changed, 6 insertions(+), 32 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java index ac16758d1..52509447d 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java @@ -854,7 +854,7 @@ public class AclTracker extends AbstractTracker private List filterAcls(List acls) { - ArrayList filteredList = new ArrayList(acls.size()); + ArrayList filteredList = new ArrayList(acls.size()); for(Acl acl : acls) { if(docRouter.routeAcl(shardCount, shardInstance, acl)) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java index ca151a5b1..363206404 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java @@ -139,8 +139,8 @@ public class CascadeTracker extends AbstractTracker implements Tracker return; } - ArrayList txIds = new ArrayList(); - Set txIdSet = new HashSet(); + ArrayList txIds = new ArrayList(); + Set txIdSet = new HashSet(); for (Transaction tx : txBatch) { txIds.add(tx.getId()); txIdSet.add(tx.getId()); @@ -150,12 +150,12 @@ public class CascadeTracker extends AbstractTracker implements Tracker //System.out.println("########### Cascade node meta datas:"+nodeMetaDatas.size()); if(nodeMetaDatas.size() > 0) { - LinkedList stack = new LinkedList(); + LinkedList stack = new LinkedList(); stack.addAll(nodeMetaDatas); int batchSize = 10; do { - List batch = new ArrayList(); + List batch = new ArrayList(); while (batch.size() < batchSize && stack.size() > 0) { batch.add(stack.removeFirst()); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 253bc2559..091b1bd6e 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -24,7 +24,6 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Properties; -import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import org.alfresco.error.AlfrescoRuntimeException; @@ -1066,26 +1065,6 @@ public class MetadataTracker extends AbstractTracker implements Tracker this.queriesToReindex.offer(query); } - private boolean transactionDocumentIdProcessed(Set transactionDocumentIdSet, Long id) throws IOException - { - if(transactionDocumentIdSet.contains(id)) - { - return true; - } - else - { - if(infoSrv.isInIndex(Long.toString(id))) { - return true; - } - else - { - transactionDocumentIdSet.add(id); - return false; - } - } - - } - public static QName getShardProperty(String field) { AlfrescoSolrDataModel dataModel = AlfrescoSolrDataModel.getInstance(); NamespaceDAO namespaceDAO = dataModel.getNamespaceDAO(); @@ -1097,8 +1076,4 @@ public class MetadataTracker extends AbstractTracker implements Tracker return propertyDef.getName(); } - - - - } diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index d77ded20e..13db1f42d 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -50,8 +50,7 @@ alfresco.encryption.ssl.truststore.provider= alfresco.encryption.ssl.truststore.location=ssl.repo.client.truststore alfresco.encryption.ssl.truststore.passwordFileLocation=ssl-truststore-passwords.properties -# Tracking //Deprecate this and push it down to tracker - +# Tracking alfresco.corePoolSize=8 alfresco.maximumPoolSize=-1 alfresco.keepAliveTime=120 From 5b3207a293ee951ca53dbaffe3e6657a63a0d705 Mon Sep 17 00:00:00 2001 From: Michael Suzuki Date: Tue, 27 Jun 2017 11:39:00 +0100 Subject: [PATCH 6/7] Search-77, seperating configuration of thread pools to 3 key trackers --- .../pool/DefaultTrackerPoolFactory.java | 33 ++++++++ .../templates/rerank/conf/solrcore.properties | 27 ++++++- .../solr/tracker/AlfrescoSolrTrackerTest.java | 10 ++- .../pool/DefaultTrackerPoolFactoryTest.java | 75 ++++++++++++++++++- 4 files changed, 141 insertions(+), 4 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java index ad377efc4..6e42800b7 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java @@ -75,6 +75,39 @@ public class DefaultTrackerPoolFactory implements TrackerPoolFactory threadPriority = Integer.parseInt(p.getProperty("alfresco.threadPriority", "5")); threadDaemon = Boolean.parseBoolean(p.getProperty("alfresco.threadDaemon", "true")); workQueueSize = Integer.parseInt(p.getProperty("alfresco.workQueueSize", "-1")); + if(trackerName != null && !trackerName.isEmpty()) + { + switch (trackerName) + { + case "AclTracker": + corePoolSize = Integer.parseInt(p.getProperty("alfresco.acl.tracker.corePoolSize", "4")); + maximumPoolSize = Integer.parseInt(p.getProperty("alfresco.acl.tracker.maximumPoolSize", "10")); + keepAliveTime = Integer.parseInt(p.getProperty("alfresco.acl.tracker.keepAliveTime", "120")); + threadPriority = Integer.parseInt(p.getProperty("alfresco.acl.tracker.threadPriority", "6")); + threadDaemon = Boolean.parseBoolean(p.getProperty("alfresco.acl.tracker.threadDaemon", "true")); + workQueueSize = Integer.parseInt(p.getProperty("alfresco.acl.tracker.workQueueSize", "-1")); + break; + case "ContentTracker": + corePoolSize = Integer.parseInt(p.getProperty("alfresco.content.tracker.corePoolSize", "12")); + maximumPoolSize = Integer.parseInt(p.getProperty("alfresco.content.tracker.maximumPoolSize", "-1")); + keepAliveTime = Integer.parseInt(p.getProperty("alfresco.content.tracker.keepAliveTime", "120")); + threadPriority = Integer.parseInt(p.getProperty("alfresco.content.tracker.threadPriority", "4")); + threadDaemon = Boolean.parseBoolean(p.getProperty("alfresco.content.tracker.threadDaemon", "true")); + workQueueSize = Integer.parseInt(p.getProperty("alfresco.content.tracker.workQueueSize", "-1")); + break; + case "MetadataTracker": + corePoolSize = Integer.parseInt(p.getProperty("alfresco.metadata.tracker.corePoolSize", "5")); + maximumPoolSize = Integer.parseInt(p.getProperty("alfresco.metadata.tracker.maximumPoolSize", "-1")); + keepAliveTime = Integer.parseInt(p.getProperty("alfresco.metadata.tracker.keepAliveTime", "120")); + threadPriority = Integer.parseInt(p.getProperty("alfresco.metadata.tracker.threadPriority", "5")); + threadDaemon = Boolean.parseBoolean(p.getProperty("alfresco.metadata.tracker.threadDaemon", "true")); + workQueueSize = Integer.parseInt(p.getProperty("alfresco.metadata.tracker.workQueueSize", "-1")); + break; + default: + break; + } + } + poolName = "SolrTrackingPool-" + coreName + "-" + trackerName + "-"; diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index 13db1f42d..d4691d5bb 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -50,18 +50,41 @@ alfresco.encryption.ssl.truststore.provider= alfresco.encryption.ssl.truststore.location=ssl.repo.client.truststore alfresco.encryption.ssl.truststore.passwordFileLocation=ssl-truststore-passwords.properties -# Tracking +# Default Tracker alfresco.corePoolSize=8 alfresco.maximumPoolSize=-1 alfresco.keepAliveTime=120 alfresco.threadPriority=5 alfresco.threadDaemon=true alfresco.workQueueSize=-1 -#Track configuration +#ACL tracker configuration alfresco.acl.tracker.cron=0/15 * * * * ? * +alfresco.acl.tracker.corePoolSize=4 +alfresco.acl.tracker.maximumPoolSize=10 +alfresco.acl.tracker.keepAliveTime=120 +alfresco.acl.tracker.threadPriority=6 +alfresco.acl.tracker.threadDaemon=true +alfresco.acl.tracker.workQueueSize=-1 +#Content tracker config alfresco.content.tracker.cron=0/15 * * * * ? * +alfresco.content.tracker.corePoolSize=12 +alfresco.content.tracker.maximumPoolSize=-1 +alfresco.content.tracker.keepAliveTime=120 +alfresco.content.tracker.threadPriority=4 +alfresco.content.tracker.threadDaemon=true +alfresco.content.tracker.workQueueSize=-1 +#Metadata tracker config alfresco.metadata.tracker.cron=0/15 * * * * ? * +alfresco.metadata.tracker.corePoolSize=5 +alfresco.metadata.tracker.maximumPoolSize=-1 +alfresco.metadata.tracker.keepAliveTime=120 +alfresco.metadata.tracker.threadPriority=5 +alfresco.metadata.tracker.threadDaemon=true +alfresco.metadata.tracker.workQueueSize=-1 +#Cascade tracker config alfresco.cascade.tracker.cron=0/15 * * * * ? * + + alfresco.commit.tracker.cron=0/15 * * * * ? * alfresco.model.tracker.cron=0/15 * * * * ? * alfresco.commitInterval=2000 diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerTest.java index 6d5cfcb26..c1e8c70fb 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerTest.java @@ -38,7 +38,15 @@ import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.solr.AbstractAlfrescoSolrTests; -import org.alfresco.solr.client.*; +import org.alfresco.solr.client.Acl; +import org.alfresco.solr.client.AclChangeSet; +import org.alfresco.solr.client.AclReaders; +import org.alfresco.solr.client.ContentPropertyValue; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.SOLRAPIQueueClient; +import org.alfresco.solr.client.StringPropertyValue; +import org.alfresco.solr.client.Transaction; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.index.Term; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java index 1270ae06e..2666a743b 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java @@ -42,7 +42,7 @@ public class DefaultTrackerPoolFactoryTest public void setup() { poolFactory = null; // Ensure we don't accidentally reuse between runs. - properties = new Properties(); + properties = new Properties(); } @After public void teardown() @@ -78,4 +78,77 @@ public class DefaultTrackerPoolFactoryTest assertEquals(40, tpe.getMaximumPoolSize()); assertEquals(200, tpe.getKeepAliveTime(TimeUnit.SECONDS)); } + @Test + public void testAclDefaultProperties() + { + poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "AclTracker"); + + tpe = poolFactory.create(); + + assertEquals(4, tpe.getCorePoolSize()); + assertEquals(10, tpe.getMaximumPoolSize()); + assertEquals(120, tpe.getKeepAliveTime(TimeUnit.SECONDS)); + } + @Test + public void testAclProperties() + { + properties.put("alfresco.acl.tracker.corePoolSize", "30"); + properties.put("alfresco.acl.tracker.maximumPoolSize", "40"); + properties.put("alfresco.acl.tracker.keepAliveTime", "200"); + poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "AclTracker"); + tpe = poolFactory.create(); + + assertEquals(30, tpe.getCorePoolSize()); + assertEquals(40, tpe.getMaximumPoolSize()); + assertEquals(200, tpe.getKeepAliveTime(TimeUnit.SECONDS)); + } + @Test + public void testContentDefaultProperties() + { + poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "ContentTracker"); + + tpe = poolFactory.create(); + + assertEquals(12, tpe.getCorePoolSize()); + assertEquals(12, tpe.getMaximumPoolSize()); + assertEquals(120, tpe.getKeepAliveTime(TimeUnit.SECONDS)); + } + @Test + public void testContentProperties() + { + properties.put("alfresco.content.tracker.corePoolSize", "100"); + properties.put("alfresco.content.tracker.maximumPoolSize", "140"); + properties.put("alfresco.content.tracker.keepAliveTime", "201"); + poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "ContentTracker"); + tpe = poolFactory.create(); + + assertEquals(100, tpe.getCorePoolSize()); + assertEquals(140, tpe.getMaximumPoolSize()); + assertEquals(201, tpe.getKeepAliveTime(TimeUnit.SECONDS)); + } + @Test + public void testMetaDataDefaultProperties() + { + poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "MetadataTracker"); + + tpe = poolFactory.create(); + + assertEquals(5, tpe.getCorePoolSize()); + assertEquals(5, tpe.getMaximumPoolSize()); + assertEquals(120, tpe.getKeepAliveTime(TimeUnit.SECONDS)); + } + @Test + public void testMetaDataProperties() + { + properties.put("alfresco.metadata.tracker.corePoolSize", "100"); + properties.put("alfresco.metadata.tracker.maximumPoolSize", "140"); + properties.put("alfresco.metadata.tracker.keepAliveTime", "201"); + poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "MetadataTracker"); + + tpe = poolFactory.create(); + + assertEquals(100, tpe.getCorePoolSize()); + assertEquals(140, tpe.getMaximumPoolSize()); + assertEquals(201, tpe.getKeepAliveTime(TimeUnit.SECONDS)); + } } From c58b99dbd8bfc310affea1c263ab972b9f05ced7 Mon Sep 17 00:00:00 2001 From: Michael Suzuki Date: Fri, 30 Jun 2017 08:51:24 +0100 Subject: [PATCH 7/7] Search-77, including shared.properties to SolrCoreLoadRegistration --- .../java/org/alfresco/solr/SolrInformationServer.java | 9 ++++++++- .../solr/lifecycle/SolrCoreLoadRegistration.java | 4 +--- .../main/java/org/alfresco/solr/tracker/AclTracker.java | 2 +- .../main/resources/solr/instance/conf/shared.properties | 4 +++- .../instance/templates/rerank/conf/solrcore.properties | 2 -- .../alfresco/solr/tracker/SolrTrackerSchedulerTest.java | 4 ---- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 21fabd950..eaa4e3c25 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -267,6 +267,7 @@ public class SolrInformationServer implements InformationServer private String hostName; private String baseUrl; + Properties props; private long cleanContentTxnFloor = -1; // All transactions below this floor have had the content completed. private long cleanCascadeTxnFloor = -1; @@ -313,7 +314,7 @@ public class SolrInformationServer implements InformationServer contentStreamLimit = Integer.parseInt(p.getProperty("alfresco.contentStreamLimit", "10000000")); // build base URL - host and port have to come from configuration. - Properties props = AlfrescoSolrDataModel.getCommonConfig(); + props = AlfrescoSolrDataModel.getCommonConfig(); hostName = ConfigUtil.locateProperty(SOLR_HOST, props.getProperty(SOLR_HOST)); String portNumber = ConfigUtil.locateProperty(SOLR_PORT, props.getProperty(SOLR_PORT)); @@ -4117,5 +4118,11 @@ public class SolrInformationServer implements InformationServer { return solrContentStore; } + + + public Properties getProps() + { + return props; + } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java index d9e451799..7afe6a209 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java @@ -69,11 +69,9 @@ public class SolrCoreLoadRegistration { if (Boolean.parseBoolean(props.getProperty("enable.alfresco.tracking", "false"))) { - SolrTrackerScheduler scheduler = adminHandler.getScheduler(); SolrResourceLoader loader = core.getLatestSchema().getResourceLoader(); SolrKeyResourceLoader keyResourceLoader = new SolrKeyResourceLoader(loader); - if (trackerRegistry.hasTrackersForCore(coreName)) { log.info("Trackers for " + coreName+ " is already registered, shutting them down."); @@ -89,7 +87,7 @@ public class SolrCoreLoadRegistration { //Start content store SolrContentStore contentStore = new SolrContentStore(coreContainer.getSolrHome()); SolrInformationServer srv = new SolrInformationServer(adminHandler, core, repositoryClient, contentStore); - + props.putAll(srv.getProps()); adminHandler.getInformationServers().put(coreName, srv); log.info("Starting to track " + coreName); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java index 52509447d..c2e782b6a 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java @@ -854,7 +854,7 @@ public class AclTracker extends AbstractTracker private List filterAcls(List acls) { - ArrayList filteredList = new ArrayList(acls.size()); + ArrayList filteredList = new ArrayList(acls.size()); for(Acl acl : acls) { if(docRouter.routeAcl(shardCount, shardInstance, acl)) diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties b/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties index e316d4606..b2c22f3d0 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties @@ -25,4 +25,6 @@ alfresco.cross.locale.property.0={http://www.alfresco.org/model/content/1.0}name # Data types that support cross locale/word splitting/token patterns if tokenised # alfresco.cross.locale.datatype.0={http://www.alfresco.org/model/dictionary/1.0}text # alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content -# alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext \ No newline at end of file +# alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext + +alfresco.model.tracker.cron=0/16 * * * * ? * \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index d4691d5bb..84b100a2e 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -84,9 +84,7 @@ alfresco.metadata.tracker.workQueueSize=-1 #Cascade tracker config alfresco.cascade.tracker.cron=0/15 * * * * ? * - alfresco.commit.tracker.cron=0/15 * * * * ? * -alfresco.model.tracker.cron=0/15 * * * * ? * alfresco.commitInterval=2000 alfresco.newSearcherInterval=3000 diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java index 4a52146a6..b97f0e7c8 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java @@ -23,17 +23,14 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import java.util.Arrays; import java.util.Properties; -import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.alfresco.solr.InformationServer; import org.alfresco.solr.client.SOLRAPIClient; @@ -210,5 +207,4 @@ public class SolrTrackerSchedulerTest verify(spiedQuartzScheduler).scheduleJob(any(JobDetail.class), any(Trigger.class)); checkCronExpression(exp); } - }