From 2e3615613827ba40004e95f347c5c8cb22f80712 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 26 Jul 2019 16:25:27 +0100 Subject: [PATCH] Revert "Merge branch 'feature/SEARCH-1738_ShardingParams' into 'master'" This reverts commit 0b375ed23afd377dd36c6a20bf74d384b2f35f91, reversing changes made to 558925e169cbaa93b3d44ae89e3c49dbb12ab830. --- .../solr/tracker/DBIDRangeRouter.java | 19 +----- .../solr/tracker/DateMonthRouter.java | 14 ----- .../org/alfresco/solr/tracker/DocRouter.java | 18 +----- .../solr/tracker/DocRouterFactory.java | 59 +++++++++---------- ...licitShardIdWithDynamicPropertyRouter.java | 14 ----- .../solr/tracker/MetadataTracker.java | 6 +- .../alfresco/solr/tracker/PropertyRouter.java | 56 ++++++++++-------- 7 files changed, 66 insertions(+), 120 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DBIDRangeRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DBIDRangeRouter.java index bdb46e53a..5f69d0b6b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DBIDRangeRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DBIDRangeRouter.java @@ -18,15 +18,11 @@ */ package org.alfresco.solr.tracker; -import java.util.Map; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.Acl; + import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.alfresco.service.namespace.QName; -import org.alfresco.solr.client.Acl; -import org.alfresco.solr.client.Node; /** * This routes documents within specific DBID ranges to specific shards. @@ -108,13 +104,4 @@ public class DBIDRangeRouter implements DocRouter long dbid = node.getId(); return dbid >= startRange && dbid < expandableRange.longValue(); } - - @Override - public Map getProperties(QName shardProperty) - { - return Stream.of(new String[][] { - { DocRouterFactory.SHARD_RANGE_KEY, startRange + "-" + expandableRange }, - }).collect(Collectors.toMap(data -> data[0], data -> data[1])); - } - } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateMonthRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateMonthRouter.java index bbd69db70..b6302b743 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateMonthRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateMonthRouter.java @@ -20,7 +20,6 @@ package org.alfresco.solr.tracker; import org.alfresco.util.ISO8601DateFormat; import org.alfresco.solr.client.Node; -import org.alfresco.service.namespace.QName; import org.alfresco.solr.client.Acl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,9 +27,6 @@ import org.slf4j.LoggerFactory; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * The date-based sharding assigns dates sequentially through shards based on the month. @@ -120,14 +116,4 @@ public class DateMonthRouter implements DocRouter return dbidRouter.routeNode(numShards, shardInstance, node); } } - - @Override - public Map getProperties(QName shardProperty) - { - return Stream.of(new String[][] { - { DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString() }, - { DocRouterFactory.SHARD_DATE_GROUPING_KEY, String.valueOf(grouping) }, - }).collect(Collectors.toMap(data -> data[0], data -> data[1])); - } - } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouter.java index 694780c4b..1370ba440 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouter.java @@ -19,11 +19,6 @@ package org.alfresco.solr.tracker; import org.alfresco.solr.client.Node; - -import java.util.Collections; -import java.util.Map; - -import org.alfresco.service.namespace.QName; import org.alfresco.solr.client.Acl; /** @@ -63,15 +58,4 @@ public interface DocRouter * @return true if the {@link Node} instance must be indexed in the shard which owns this {@link DocRouter} instance, false otherwise. */ Boolean routeNode(int shardCount, int shardInstance, Node node); - - /** - * Get additional properties to "shardProperty" depending on the Shard Method - * @param shardProperty custom property used to configure the Router - * @return pair of key, value - */ - default public Map getProperties(QName shardProperty) { - return Collections.emptyMap(); - } - -} - +} \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java index 6937a0b07..1399a6316 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java @@ -24,60 +24,59 @@ import org.slf4j.LoggerFactory; import java.util.Properties; -/* +/** + * Routing strategy Factory. + * * @author Joel */ - public class DocRouterFactory { - protected final static Logger log = LoggerFactory.getLogger(DocRouterFactory.class); - - public static final String SHARD_KEY_KEY = "shard.key"; - public static final String SHARD_RANGE_KEY = "shard.range"; - public static final String SHARD_DATE_GROUPING_KEY = "shard.date.grouping"; - public static final String SHARD_REGEX_KEY = "shard.regex"; + private final static Logger LOGGER = LoggerFactory.getLogger(DocRouterFactory.class); - public static DocRouter getRouter(Properties properties, ShardMethodEnum method) { - - switch(method) { + public static DocRouter getRouter(Properties properties, ShardMethodEnum method) + { + switch(method) + { case DB_ID: - log.info("Sharding via DB_ID"); + LOGGER.info("Sharding via DB_ID"); return new DBIDRouter(); case DB_ID_RANGE: - // - if(properties.containsKey(SHARD_RANGE_KEY)) + if(properties.containsKey("shard.range")) { - log.info("Sharding via DB_ID_RANGE"); - String[] pair =properties.getProperty(SHARD_RANGE_KEY).split("-"); + LOGGER.info("Sharding via DB_ID_RANGE"); + String[] pair =properties.getProperty("shard.range").split("-"); long start = Long.parseLong(pair[0]); long end = Long.parseLong(pair[1]); return new DBIDRangeRouter(start, end); } case ACL_ID: - log.info("Sharding via ACL_ID"); + LOGGER.info("Sharding via ACL_ID"); return new ACLIDMurmurRouter(); case MOD_ACL_ID: - log.info("Sharding via MOD_ACL_ID"); + LOGGER.info("Sharding via MOD_ACL_ID"); return new ACLIDModRouter(); case DATE: - log.info("Sharding via DATE"); - return new DateMonthRouter(properties.getProperty(SHARD_DATE_GROUPING_KEY, "1")); + LOGGER.info("Sharding via DATE"); + return new DateMonthRouter(properties.getProperty("shard.date.grouping", "1")); case PROPERTY: - log.info("Sharding via PROPERTY"); - return new PropertyRouter(properties.getProperty(SHARD_REGEX_KEY, "")); + LOGGER.info("Sharding via PROPERTY"); + return new PropertyRouter(properties.getProperty("shard.regex", "")); case LAST_REGISTERED_INDEXING_SHARD: - log.info("Sharding via LAST_REGISTERED_INDEXING_SHARD"); - return new LastRegisteredShardRouter(); + LOGGER.info("Sharding via LAST_REGISTERED_INDEXING_SHARD"); + return new ExplicitShardIdWithStaticPropertyRouter(); case EXPLICIT_ID_FALLBACK_LRIS: - log.info("Sharding via EXPLICIT_ID_FALLBACK_LRIS"); - return new ExplicitRouter(new LastRegisteredShardRouter()); + LOGGER.info("Sharding via EXPLICIT_ID_FALLBACK_LRIS"); + return new DocRouterWithFallback( + new ExplicitShardIdWithDynamicPropertyRouter(false), + new ExplicitShardIdWithStaticPropertyRouter()); case EXPLICIT_ID: - log.info("Sharding via EXPLICIT_ID"); - return new ExplicitRouter(new DBIDRouter()); + LOGGER.info("Sharding via EXPLICIT_ID"); + return new DocRouterWithFallback( + new ExplicitShardIdWithDynamicPropertyRouter(false), + new DBIDRouter()); default: - log.info("Sharding via DB_ID (default)"); + LOGGER.warn("WARNING! Unknown/unsupported sharding method ({}). System will fallback to DB_ID", method); return new DBIDRouter(); } } - } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithDynamicPropertyRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithDynamicPropertyRouter.java index ed4e6d154..ad208df53 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithDynamicPropertyRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithDynamicPropertyRouter.java @@ -18,11 +18,6 @@ */ package org.alfresco.solr.tracker; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.alfresco.service.namespace.QName; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.Node; @@ -77,13 +72,4 @@ public class ExplicitShardIdWithDynamicPropertyRouter extends ComposableDocRoute return negativeReturnValue(); } } - - @Override - public Map getProperties(QName shardProperty) - { - return Stream.of(new String[][] { - { DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString() }, - }).collect(Collectors.toMap(data -> data[0], data -> data[1])); - } - } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index b40ba1edd..c35d46267 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -86,10 +86,11 @@ public class MetadataTracker extends AbstractTracker implements Tracker super(p, client, coreName, informationServer, Tracker.Type.MetaData); transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); - String shardKey = p.getProperty(DocRouterFactory.SHARD_KEY_KEY); + String shardKey = p.getProperty("shard.key"); if(shardKey != null) { shardProperty = getShardProperty(shardKey); } + docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod)); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); @@ -223,8 +224,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker HashMap propertyBag = new HashMap<>(); propertyBag.put("coreName", coreName); - propertyBag.putAll(docRouter.getProperties(shardProperty)); - + return ShardStateBuilder.shardState() .withMaster(isMaster) .withLastUpdated(System.currentTimeMillis()) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/PropertyRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/PropertyRouter.java index 668265067..9db9b6dab 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/PropertyRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/PropertyRouter.java @@ -19,58 +19,72 @@ package org.alfresco.solr.tracker; import org.alfresco.repo.index.shard.ShardMethodEnum; -import org.alfresco.service.namespace.QName; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.Node; import org.apache.solr.common.util.Hash; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Routes based on a text property field. + * In this method, the value of some property is hashed and this hash is used to assign the node to a random shard. + * All nodes with the same property value will be assigned to the same shard. + * Each shard will duplicate all the ACL information. + * + * To use this method, when creating a shard add the new configuration properties: + * + *
    + *
  • shard.key=cm:creator
    • + *
  • shard.method=PROPERTY
    • + *
  • shard.instance=<shard.instance>
    • + *
  • shard.count=<shard.count>
    • + *
+ * + * It is possible to extract a part of the property value to use for sharding using a regular expression, + * for example, a year at the start of a string: + * + *
    + *
  • shard.regex=^\d{4}
    • + *
* * @author Gethin James + * @see Search Services sharding methods */ public class PropertyRouter implements DocRouter { - protected final static Logger log = LoggerFactory.getLogger(PropertyRouter.class); + private final static Logger LOGGER = LoggerFactory.getLogger(PropertyRouter.class); - Pattern pattern = null; - private String propertyRegEx; + Pattern pattern; //Fallback to DB_ID routing - private DocRouter fallback = DocRouterFactory.getRouter(null, ShardMethodEnum.DB_ID); + DocRouter fallback = DocRouterFactory.getRouter(null, ShardMethodEnum.DB_ID); public PropertyRouter(String propertyRegEx) { - if (propertyRegEx != null && !propertyRegEx.isEmpty()) + if (propertyRegEx != null && propertyRegEx.trim().length() > 0) { - this.propertyRegEx = propertyRegEx; - pattern = Pattern.compile(propertyRegEx); + pattern = Pattern.compile(propertyRegEx.trim()); } } @Override - public boolean routeAcl(int shardCount, int shardInstance, Acl acl) + public Boolean routeAcl(int shardCount, int shardInstance, Acl acl) { return true; } @Override - public boolean routeNode(int shardCount, int shardInstance, Node node) + public Boolean routeNode(int shardCount, int shardInstance, Node node) { if(shardCount <= 1) { return true; } - String shardBy = node.getShardPropertyValue(); + String shardBy = node.getShardPropertyValue(); if (shardBy !=null && pattern != null) { try @@ -88,27 +102,17 @@ public class PropertyRouter implements DocRouter } catch (IndexOutOfBoundsException | NullPointerException exc) { - log.debug("Regex matched, but group 1 not found, so falling back to DBID sharding."); + LOGGER.debug("Regex matched, but group 1 not found, so falling back to DBID sharding."); shardBy = null; } } if (shardBy == null || shardBy.isEmpty()) { - log.debug("Property not found or regex not matched, so falling back to DBID sharding."); + LOGGER.debug("Property not found or regex not matched, so falling back to DBID sharding."); return fallback.routeNode(shardCount,shardInstance,node); } return (Math.abs(Hash.murmurhash3_x86_32(shardBy, 0, shardBy.length(), 66)) % shardCount) == shardInstance; } - - @Override - public Map getProperties(QName shardProperty) - { - return Stream.of(new String[][] { - { DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString() }, - { DocRouterFactory.SHARD_REGEX_KEY, propertyRegEx }, - }).collect(Collectors.toMap(data -> data[0], data -> data[1])); - } - }