From 2dadf882d90e5d5c03d023b89268c04a373ba09a Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 22 Aug 2019 12:47:26 +0200 Subject: [PATCH 1/3] [SEARCH-960] ModelTracker refactoring --- .../alfresco/solr/tracker/ModelTracker.java | 99 +++++++------------ 1 file changed, 37 insertions(+), 62 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java index 687fb4077..cc0d2577f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java @@ -83,18 +83,18 @@ import org.json.JSONException; public class ModelTracker extends AbstractTracker implements Tracker { - private Set indexedStores = new HashSet(); - private Set ignoredStores = new HashSet(); - private Set indexedTenants = new HashSet(); - private Set ignoredTenants = new HashSet(); - private Set indexedDataTypes = new HashSet(); - private Set ignoredDataTypes = new HashSet(); - private Set indexedTypes = new HashSet(); - private Set ignoredTypes = new HashSet(); - private Set indexedAspects = new HashSet(); - private Set ignoredAspects = new HashSet(); - private Set indexedFields = new HashSet(); - private Set ignoredFields = new HashSet(); + private final Set indexedStores = new HashSet<>(); + private final Set ignoredStores = new HashSet<>(); + private final Set indexedTenants = new HashSet<>(); + private final Set ignoredTenants = new HashSet<>(); + private final Set indexedDataTypes = new HashSet<>(); + private final Set ignoredDataTypes = new HashSet<>(); + private final Set indexedTypes = new HashSet<>(); + private final Set ignoredTypes = new HashSet<>(); + private final Set indexedAspects = new HashSet<>(); + private final Set ignoredAspects = new HashSet<>(); + private final Set indexedFields = new HashSet<>(); + private final Set ignoredFields = new HashSet<>(); private ReentrantReadWriteLock modelLock = new ReentrantReadWriteLock(); private volatile boolean hasModels = false; @@ -111,7 +111,7 @@ public class ModelTracker extends AbstractTracker implements Tracker { alfrescoModelDir.mkdir(); } - + loadPersistedModels(); } @@ -124,23 +124,15 @@ public class ModelTracker extends AbstractTracker implements Tracker } /** - * + * */ private void loadPersistedModels() { - HashMap modelMap = new HashMap(); + HashMap modelMap = new HashMap<>(); if (alfrescoModelDir.exists() && alfrescoModelDir.isDirectory()) { // A filter for XML files - FileFilter filter = new FileFilter() - { - @Override - public boolean accept(File pathname) - { - return pathname.isFile() && pathname.getName().endsWith(".xml"); - } - - }; + FileFilter filter = pathname -> pathname.isFile() && pathname.getName().endsWith(".xml"); // List XML files File[] files = alfrescoModelDir.listFiles(filter); if (files != null) @@ -289,26 +281,20 @@ public class ModelTracker extends AbstractTracker implements Tracker long start = System.nanoTime(); List modelDiffs = client.getModelsDiff(coreName, this.infoSrv.getAlfrescoModels()); - HashMap modelMap = new HashMap(); + Map modelMap = new HashMap<>(); for (AlfrescoModelDiff modelDiff : modelDiffs) { switch (modelDiff.getType()) { case CHANGED: + case NEW: AlfrescoModel changedModel = client.getModel(coreName, modelDiff.getModelName()); for (M2Namespace namespace : changedModel.getModel().getNamespaces()) { modelMap.put(namespace.getUri(), changedModel.getModel()); } break; - case NEW: - AlfrescoModel newModel = client.getModel(coreName, modelDiff.getModelName()); - for (M2Namespace namespace : newModel.getModel().getNamespaces()) - { - modelMap.put(namespace.getUri(), newModel.getModel()); - } - break; case REMOVED: // We now remove models as we see them - MNT-17627 // Models have to be deleted from disk before we remove them from memory @@ -317,7 +303,7 @@ public class ModelTracker extends AbstractTracker implements Tracker } } - HashSet loadedModels = new HashSet(); + HashSet loadedModels = new HashSet<>(); for (M2Model model : modelMap.values()) { loadModel(modelMap, loadedModels, model); @@ -367,7 +353,7 @@ public class ModelTracker extends AbstractTracker implements Tracker trackerStats.addModelTime(end - start); - if (true == runPostModelLoadInit) + if (runPostModelLoadInit) { for (Object key : props.keySet()) { @@ -450,10 +436,7 @@ public class ModelTracker extends AbstractTracker implements Tracker { expandedQName = expandQNameImpl(qName); } - // else if (AlfrescoSolrDataModel.nonDictionaryFields.get(qName) == null) - // { - // expandedQName = expandQNameImpl(qName); - // } + return QName.createQName(expandedQName); } @@ -495,10 +478,7 @@ public class ModelTracker extends AbstractTracker implements Tracker { expandedQName = expandQNameImpl(qName); } - // else if (AlfrescoSolrDataModel.nonDictionaryFields.get(qName) == null) - // { - // expandedQName = expandQNameImpl(qName); - // } + return expandedQName; } @@ -513,26 +493,21 @@ public class ModelTracker extends AbstractTracker implements Tracker final String prefix = modelName.toPrefixString(this.infoSrv.getNamespaceDAO()).replace(":", ".") + "."; final String postFix = ".xml"; - File[] toDelete = alfrescoModelDir.listFiles(new FileFilter() - { - @Override - public boolean accept(File pathname) + File[] toDelete = alfrescoModelDir.listFiles(pathname -> { + if (pathname.isDirectory()) { return false; } + String name = pathname.getName(); + if (!name.endsWith(postFix)) { return false; } + if (!name.startsWith(prefix)) { return false; } + // check is number between + String checksum = name.substring(prefix.length(), name.length() - postFix.length()); + try { - if (pathname.isDirectory()) { return false; } - String name = pathname.getName(); - if (false == name.endsWith(postFix)) { return false; } - if (false == name.startsWith(prefix)) { return false; } - // check is number between - String checksum = name.substring(prefix.length(), name.length() - postFix.length()); - try - { - Long.parseLong(checksum); - return true; - } - catch (NumberFormatException nfe) - { - return false; - } + Long.parseLong(checksum); + return true; + } + catch (NumberFormatException nfe) + { + return false; } }); @@ -548,7 +523,7 @@ public class ModelTracker extends AbstractTracker implements Tracker private void loadModel(Map modelMap, HashSet loadedModels, M2Model model) { String modelName = model.getName(); - if (loadedModels.contains(modelName) == false) + if (!loadedModels.contains(modelName)) { for (M2Namespace importNamespace : model.getImports()) { From b44c0fd02b222a6a218fb4adbae9552c812edb53 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 22 Aug 2019 12:48:19 +0200 Subject: [PATCH 2/3] [SEARCH-960] Run ModelTracker even when tracking for all cores is disabled --- .../lifecycle/SolrCoreLoadRegistration.java | 89 +++++++++++++------ 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java index ea4820ef0..c7f46f770 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java @@ -64,6 +64,8 @@ public class SolrCoreLoadRegistration { public static void registerForCore(AlfrescoCoreAdminHandler adminHandler, CoreContainer coreContainer, SolrCore core, String coreName) { + + TrackerRegistry trackerRegistry = adminHandler.getTrackerRegistry(); Properties props = new CoreDescriptorDecorator(core.getCoreDescriptor()).getProperties(); //Prepare cores @@ -79,10 +81,23 @@ public class SolrCoreLoadRegistration { props.putAll(srv.getProps()); adminHandler.getInformationServers().put(coreName, srv); + SolrTrackerScheduler scheduler = adminHandler.getScheduler(); + + // Prevents other threads from registering the ModelTracker at the same time + // Create model tracker and load all the persisted models + createModelTracker(coreName, + trackerRegistry, + props, + coreContainer.getSolrHome(), + repositoryClient, + srv, + scheduler); + + log.info("Starting to track " + coreName); if (Boolean.parseBoolean(props.getProperty("enable.alfresco.tracking", "false"))) { - SolrTrackerScheduler scheduler = adminHandler.getScheduler(); + if (trackerRegistry.hasTrackersForCore(coreName)) { log.info("Trackers for " + coreName+ " is already registered, shutting them down."); @@ -91,28 +106,7 @@ public class SolrCoreLoadRegistration { adminHandler.getInformationServers().remove(coreName); } - // Prevents other threads from registering the ModelTracker at the same time - synchronized (SolrCoreLoadRegistration.class) - { - ModelTracker mTracker = trackerRegistry.getModelTracker(); - if (mTracker == null) - { - log.debug("Creating ModelTracker when registering trackers for core " + coreName); - mTracker = new ModelTracker(coreContainer.getSolrHome(), props, repositoryClient, - coreName, srv); - - trackerRegistry.setModelTracker(mTracker); - - log.info("Ensuring first model sync."); - mTracker.ensureFirstModelSync(); - log.info("Done ensuring first model sync."); - - //Scheduling the ModelTracker. - scheduler.schedule(mTracker, coreName, props); - } - } - - List trackers = createTrackers(coreName, trackerRegistry, props, scheduler, repositoryClient, srv); + List trackers = createCoreTrackers(coreName, trackerRegistry, props, scheduler, repositoryClient, srv); CommitTracker commitTracker = new CommitTracker(props, repositoryClient, coreName, srv, trackers); trackerRegistry.register(coreName, commitTracker); @@ -120,7 +114,8 @@ public class SolrCoreLoadRegistration { log.info("The Trackers are now scheduled to run"); trackers.add(commitTracker); //Add the commitTracker to the list of scheduled trackers that can be shutdown - core.addCloseHook(new CloseHook() { + core.addCloseHook(new CloseHook() + { @Override public void preClose(SolrCore core) { @@ -148,7 +143,12 @@ public class SolrCoreLoadRegistration { * @param srv * @return A list of trackers */ - private static List createTrackers(String coreName, TrackerRegistry trackerRegistry, Properties props, SolrTrackerScheduler scheduler, SOLRAPIClient repositoryClient, SolrInformationServer srv) { + private static List createCoreTrackers(String coreName, + TrackerRegistry trackerRegistry, + Properties props, + SolrTrackerScheduler scheduler, + SOLRAPIClient repositoryClient, + SolrInformationServer srv) { List trackers = new ArrayList(); AclTracker aclTracker = new AclTracker(props, repositoryClient, coreName, srv); @@ -178,6 +178,45 @@ public class SolrCoreLoadRegistration { return trackers; } + + /** + * Create model tracker and load persisted models. + * + * @param coreName + * @param trackerRegistry + * @param props + * @param solrHome + * @param repositoryClient + * @param srv + * @param scheduler + * @return true if model tracker has been created, false if it already exists. + */ + private synchronized static void createModelTracker(String coreName, + TrackerRegistry trackerRegistry, + Properties props, + String solrHome, + SOLRAPIClient repositoryClient, + SolrInformationServer srv, + SolrTrackerScheduler scheduler) + { + ModelTracker mTracker = trackerRegistry.getModelTracker(); + if (mTracker == null) + { + log.debug("Creating ModelTracker"); + mTracker = new ModelTracker(solrHome, props, repositoryClient, + coreName, srv); + + trackerRegistry.setModelTracker(mTracker); + log.info("Ensuring first model sync."); + mTracker.ensureFirstModelSync(); + log.info("Done ensuring first model sync."); + + //Scheduling the ModelTracker. + scheduler.schedule(mTracker, coreName, props); + } + + } + /** * Shuts down the trackers for a core. * From c206fa8b63a861376621bb024296ada5e8726812 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 22 Aug 2019 14:44:36 +0200 Subject: [PATCH 3/3] [SEARCH-960] changed variable name --- .../main/java/org/alfresco/solr/tracker/ModelTracker.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java index cc0d2577f..ba4e00fa9 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java @@ -289,10 +289,10 @@ public class ModelTracker extends AbstractTracker implements Tracker { case CHANGED: case NEW: - AlfrescoModel changedModel = client.getModel(coreName, modelDiff.getModelName()); - for (M2Namespace namespace : changedModel.getModel().getNamespaces()) + AlfrescoModel model = client.getModel(coreName, modelDiff.getModelName()); + for (M2Namespace namespace : model.getModel().getNamespaces()) { - modelMap.put(namespace.getUri(), changedModel.getModel()); + modelMap.put(namespace.getUri(), model.getModel()); } break; case REMOVED: