diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DBIDRangeRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DBIDRangeRouter.java index 5f69d0b6b..bdb46e53a 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DBIDRangeRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DBIDRangeRouter.java @@ -18,11 +18,15 @@ */ package org.alfresco.solr.tracker; -import org.alfresco.solr.client.Node; -import org.alfresco.solr.client.Acl; - +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.alfresco.service.namespace.QName; +import org.alfresco.solr.client.Acl; +import org.alfresco.solr.client.Node; /** * This routes documents within specific DBID ranges to specific shards. @@ -104,4 +108,13 @@ public class DBIDRangeRouter implements DocRouter long dbid = node.getId(); return dbid >= startRange && dbid < expandableRange.longValue(); } + + @Override + public Map getProperties(QName shardProperty) + { + return Stream.of(new String[][] { + { DocRouterFactory.SHARD_RANGE_KEY, startRange + "-" + expandableRange }, + }).collect(Collectors.toMap(data -> data[0], data -> data[1])); + } + } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateMonthRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateMonthRouter.java index b6302b743..bbd69db70 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateMonthRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateMonthRouter.java @@ -20,6 +20,7 @@ package org.alfresco.solr.tracker; import org.alfresco.util.ISO8601DateFormat; import org.alfresco.solr.client.Node; +import org.alfresco.service.namespace.QName; import org.alfresco.solr.client.Acl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +28,9 @@ import org.slf4j.LoggerFactory; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * The date-based sharding assigns dates sequentially through shards based on the month. @@ -116,4 +120,14 @@ public class DateMonthRouter implements DocRouter return dbidRouter.routeNode(numShards, shardInstance, node); } } + + @Override + public Map getProperties(QName shardProperty) + { + return Stream.of(new String[][] { + { DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString() }, + { DocRouterFactory.SHARD_DATE_GROUPING_KEY, String.valueOf(grouping) }, + }).collect(Collectors.toMap(data -> data[0], data -> data[1])); + } + } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouter.java index 1370ba440..694780c4b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouter.java @@ -19,6 +19,11 @@ package org.alfresco.solr.tracker; import org.alfresco.solr.client.Node; + +import java.util.Collections; +import java.util.Map; + +import org.alfresco.service.namespace.QName; import org.alfresco.solr.client.Acl; /** @@ -58,4 +63,15 @@ public interface DocRouter * @return true if the {@link Node} instance must be indexed in the shard which owns this {@link DocRouter} instance, false otherwise. */ Boolean routeNode(int shardCount, int shardInstance, Node node); -} \ No newline at end of file + + /** + * Get additional properties to "shardProperty" depending on the Shard Method + * @param shardProperty custom property used to configure the Router + * @return pair of key, value + */ + default public Map getProperties(QName shardProperty) { + return Collections.emptyMap(); + } + +} + diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java index 1399a6316..6937a0b07 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java @@ -24,59 +24,60 @@ import org.slf4j.LoggerFactory; import java.util.Properties; -/** - * Routing strategy Factory. - * +/* * @author Joel */ + public class DocRouterFactory { - private final static Logger LOGGER = LoggerFactory.getLogger(DocRouterFactory.class); + protected final static Logger log = LoggerFactory.getLogger(DocRouterFactory.class); + + public static final String SHARD_KEY_KEY = "shard.key"; + public static final String SHARD_RANGE_KEY = "shard.range"; + public static final String SHARD_DATE_GROUPING_KEY = "shard.date.grouping"; + public static final String SHARD_REGEX_KEY = "shard.regex"; - public static DocRouter getRouter(Properties properties, ShardMethodEnum method) - { - switch(method) - { + public static DocRouter getRouter(Properties properties, ShardMethodEnum method) { + + switch(method) { case DB_ID: - LOGGER.info("Sharding via DB_ID"); + log.info("Sharding via DB_ID"); return new DBIDRouter(); case DB_ID_RANGE: - if(properties.containsKey("shard.range")) + // + if(properties.containsKey(SHARD_RANGE_KEY)) { - LOGGER.info("Sharding via DB_ID_RANGE"); - String[] pair =properties.getProperty("shard.range").split("-"); + log.info("Sharding via DB_ID_RANGE"); + String[] pair =properties.getProperty(SHARD_RANGE_KEY).split("-"); long start = Long.parseLong(pair[0]); long end = Long.parseLong(pair[1]); return new DBIDRangeRouter(start, end); } case ACL_ID: - LOGGER.info("Sharding via ACL_ID"); + log.info("Sharding via ACL_ID"); return new ACLIDMurmurRouter(); case MOD_ACL_ID: - LOGGER.info("Sharding via MOD_ACL_ID"); + log.info("Sharding via MOD_ACL_ID"); return new ACLIDModRouter(); case DATE: - LOGGER.info("Sharding via DATE"); - return new DateMonthRouter(properties.getProperty("shard.date.grouping", "1")); + log.info("Sharding via DATE"); + return new DateMonthRouter(properties.getProperty(SHARD_DATE_GROUPING_KEY, "1")); case PROPERTY: - LOGGER.info("Sharding via PROPERTY"); - return new PropertyRouter(properties.getProperty("shard.regex", "")); + log.info("Sharding via PROPERTY"); + return new PropertyRouter(properties.getProperty(SHARD_REGEX_KEY, "")); case LAST_REGISTERED_INDEXING_SHARD: - LOGGER.info("Sharding via LAST_REGISTERED_INDEXING_SHARD"); - return new ExplicitShardIdWithStaticPropertyRouter(); + log.info("Sharding via LAST_REGISTERED_INDEXING_SHARD"); + return new LastRegisteredShardRouter(); case EXPLICIT_ID_FALLBACK_LRIS: - LOGGER.info("Sharding via EXPLICIT_ID_FALLBACK_LRIS"); - return new DocRouterWithFallback( - new ExplicitShardIdWithDynamicPropertyRouter(false), - new ExplicitShardIdWithStaticPropertyRouter()); + log.info("Sharding via EXPLICIT_ID_FALLBACK_LRIS"); + return new ExplicitRouter(new LastRegisteredShardRouter()); case EXPLICIT_ID: - LOGGER.info("Sharding via EXPLICIT_ID"); - return new DocRouterWithFallback( - new ExplicitShardIdWithDynamicPropertyRouter(false), - new DBIDRouter()); + log.info("Sharding via EXPLICIT_ID"); + return new ExplicitRouter(new DBIDRouter()); default: - LOGGER.warn("WARNING! Unknown/unsupported sharding method ({}). System will fallback to DB_ID", method); + log.info("Sharding via DB_ID (default)"); return new DBIDRouter(); } } + } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithDynamicPropertyRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithDynamicPropertyRouter.java index ad208df53..ed4e6d154 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithDynamicPropertyRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithDynamicPropertyRouter.java @@ -18,6 +18,11 @@ */ package org.alfresco.solr.tracker; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.alfresco.service.namespace.QName; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.Node; @@ -72,4 +77,13 @@ public class ExplicitShardIdWithDynamicPropertyRouter extends ComposableDocRoute return negativeReturnValue(); } } + + @Override + public Map getProperties(QName shardProperty) + { + return Stream.of(new String[][] { + { DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString() }, + }).collect(Collectors.toMap(data -> data[0], data -> data[1])); + } + } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index c35d46267..b40ba1edd 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -86,11 +86,10 @@ public class MetadataTracker extends AbstractTracker implements Tracker super(p, client, coreName, informationServer, Tracker.Type.MetaData); transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); - String shardKey = p.getProperty("shard.key"); + String shardKey = p.getProperty(DocRouterFactory.SHARD_KEY_KEY); if(shardKey != null) { shardProperty = getShardProperty(shardKey); } - docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod)); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); @@ -224,7 +223,8 @@ public class MetadataTracker extends AbstractTracker implements Tracker HashMap propertyBag = new HashMap<>(); propertyBag.put("coreName", coreName); - + propertyBag.putAll(docRouter.getProperties(shardProperty)); + return ShardStateBuilder.shardState() .withMaster(isMaster) .withLastUpdated(System.currentTimeMillis()) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/PropertyRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/PropertyRouter.java index 9db9b6dab..668265067 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/PropertyRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/PropertyRouter.java @@ -19,72 +19,58 @@ package org.alfresco.solr.tracker; import org.alfresco.repo.index.shard.ShardMethodEnum; +import org.alfresco.service.namespace.QName; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.Node; import org.apache.solr.common.util.Hash; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Routes based on a text property field. - * In this method, the value of some property is hashed and this hash is used to assign the node to a random shard. - * All nodes with the same property value will be assigned to the same shard. - * Each shard will duplicate all the ACL information. - * - * To use this method, when creating a shard add the new configuration properties: - * - *
    - *
  • shard.key=cm:creator
    • - *
  • shard.method=PROPERTY
    • - *
  • shard.instance=<shard.instance>
    • - *
  • shard.count=<shard.count>
    • - *
- * - * It is possible to extract a part of the property value to use for sharding using a regular expression, - * for example, a year at the start of a string: - * - *
    - *
  • shard.regex=^\d{4}
    • - *
* * @author Gethin James - * @see Search Services sharding methods */ public class PropertyRouter implements DocRouter { - private final static Logger LOGGER = LoggerFactory.getLogger(PropertyRouter.class); + protected final static Logger log = LoggerFactory.getLogger(PropertyRouter.class); - Pattern pattern; + Pattern pattern = null; + private String propertyRegEx; //Fallback to DB_ID routing - DocRouter fallback = DocRouterFactory.getRouter(null, ShardMethodEnum.DB_ID); + private DocRouter fallback = DocRouterFactory.getRouter(null, ShardMethodEnum.DB_ID); public PropertyRouter(String propertyRegEx) { - if (propertyRegEx != null && propertyRegEx.trim().length() > 0) + if (propertyRegEx != null && !propertyRegEx.isEmpty()) { - pattern = Pattern.compile(propertyRegEx.trim()); + this.propertyRegEx = propertyRegEx; + pattern = Pattern.compile(propertyRegEx); } } @Override - public Boolean routeAcl(int shardCount, int shardInstance, Acl acl) + public boolean routeAcl(int shardCount, int shardInstance, Acl acl) { return true; } @Override - public Boolean routeNode(int shardCount, int shardInstance, Node node) + public boolean routeNode(int shardCount, int shardInstance, Node node) { if(shardCount <= 1) { return true; } - String shardBy = node.getShardPropertyValue(); + if (shardBy !=null && pattern != null) { try @@ -102,17 +88,27 @@ public class PropertyRouter implements DocRouter } catch (IndexOutOfBoundsException | NullPointerException exc) { - LOGGER.debug("Regex matched, but group 1 not found, so falling back to DBID sharding."); + log.debug("Regex matched, but group 1 not found, so falling back to DBID sharding."); shardBy = null; } } if (shardBy == null || shardBy.isEmpty()) { - LOGGER.debug("Property not found or regex not matched, so falling back to DBID sharding."); + log.debug("Property not found or regex not matched, so falling back to DBID sharding."); return fallback.routeNode(shardCount,shardInstance,node); } return (Math.abs(Hash.murmurhash3_x86_32(shardBy, 0, shardBy.length(), 66)) % shardCount) == shardInstance; } + + @Override + public Map getProperties(QName shardProperty) + { + return Stream.of(new String[][] { + { DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString() }, + { DocRouterFactory.SHARD_REGEX_KEY, propertyRegEx }, + }).collect(Collectors.toMap(data -> data[0], data -> data[1])); + } + }