From 70645cd65137a7e9948bf25581a35b031731b35e Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 8 Apr 2020 18:47:57 +0200 Subject: [PATCH 01/54] [ SEARCH-2180 ] LookAheadBufferedReader implementation + proper logging on SOLRAPIClient --- .../solr/client/LookAheadBufferedReader.java | 203 +++++++++++ .../alfresco/solr/client/SOLRAPIClient.java | 323 ++++++------------ 2 files changed, 298 insertions(+), 228 deletions(-) create mode 100644 search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java new file mode 100644 index 000000000..263074f50 --- /dev/null +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java @@ -0,0 +1,203 @@ +package org.alfresco.solr.client; + +import java.io.BufferedReader; +import java.util.LinkedList; + +import static java.util.stream.Collector.of; + +import org.slf4j.Logger; + +import java.io.IOException; +import java.io.Reader; + +/** + * This is an enhanced {@link BufferedReader} used for transparently collect data from the incoming stream before + * it gets consumed by the usual buffered reader logic. The data is not actually consumed, it is just buffered beside and + * it can be printed in case we want to debug the underlying stream content. + * + * The name refers to the usage pattern: this reader is used for wrapping a character stream coming from a remote call and, in case + * of issues, it collects the remaining (i.e. unread) part of the stream so it will be available for debugging purposes. + * + * It provides two different collecting modes: + * + * + * + * The two modes are activated on each instance depending on the level of the {@link Logger} passed in input. + * Specifically: + * + * + * + */ +public class LookAheadBufferedReader extends BufferedReader +{ + private interface BufferingMode + { + void append(char ch); + + void forceAppend(char ch); + + boolean canAccept(boolean force); + } + + private static class Windowing implements BufferingMode + { + private final LinkedList window = new LinkedList<>(); + private final int maxSize; + + private Windowing(int maxSize) + { + this.maxSize = maxSize; + } + + @Override + public void append(char ch) + { + window.add(ch); + if (window.size() == maxSize) + { + window.removeFirst(); + } + } + + @Override + public void forceAppend(char ch) + { + window.add(ch); + } + + @Override + public boolean canAccept(boolean force) + { + return window.size() < (maxSize * 2); + } + + @Override + public String toString() + { + return window.stream() + .collect( + of( + StringBuilder::new, + StringBuilder::append, + StringBuilder::append, + StringBuilder::toString)); + } + } + + private static class WholeValue implements BufferingMode + { + private final StringBuilder content = new StringBuilder(); + + @Override + public void append(char ch) + { + content.append(ch); + } + + @Override + public void forceAppend(char ch) + { + content.append(ch); + } + + @Override + public String toString() + { + return content.toString(); + } + + @Override + public boolean canAccept(boolean force) + { + return true; + } + } + + private static class NoOp implements BufferingMode + { + @Override + public void append(char ch) + { + // Nothing to be done here + } + + @Override + public void forceAppend(char ch) + { + // Nothing to be done here + } + + @Override + public boolean canAccept(boolean force) + { + return false; + } + + @Override + public String toString() + { + return "Not available: please set the logging LEVEL to DEBUG or TRACE."; + } + } + + private final BufferingMode bufferingMode; + + public LookAheadBufferedReader(Reader in, Logger logger) + { + this(in, 250, logger); + } + + public LookAheadBufferedReader(Reader in, final int windowSize, Logger logger) + { + super(in); + if (logger.isTraceEnabled()) + { + bufferingMode = new WholeValue(); + } + else if(logger.isDebugEnabled()) + { + bufferingMode = new Windowing(windowSize); + } + else { + bufferingMode = new NoOp(); + } + } + + @Override + public int read() throws IOException + { + int ch = super.read(); + bufferingMode.append((char)ch); + return ch; + } + + public String lookAheadAndGetBufferedContent() + { + try + { + int ch; + while ((ch = super.read()) != -1 && bufferingMode.canAccept(true)) + { + bufferingMode.forceAppend((char) ch); + } + } + catch (Exception ignore) + { + // Ignore any I/O exception causing further reading on the underlying stream + // Just return the collected data + } + return bufferingMode.toString(); + } +} \ No newline at end of file diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index a4cbe6b4a..c64fd87a2 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -25,11 +25,13 @@ */ package org.alfresco.solr.client; -import java.io.BufferedReader; +import static java.util.Optional.ofNullable; + +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.Reader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -46,6 +48,7 @@ import org.alfresco.httpclient.AlfrescoHttpClient; import org.alfresco.httpclient.AuthenticationException; import org.alfresco.httpclient.GetRequest; import org.alfresco.httpclient.PostRequest; +import org.alfresco.httpclient.Request; import org.alfresco.httpclient.Response; import org.alfresco.repo.dictionary.M2Model; import org.alfresco.repo.dictionary.NamespaceDAO; @@ -94,7 +97,7 @@ import com.fasterxml.jackson.core.JsonToken; */ public class SOLRAPIClient { - protected final static Logger log = LoggerFactory.getLogger(SOLRAPIClient.class); + protected final static Logger LOGGER = LoggerFactory.getLogger(SOLRAPIClient.class); private static final String GET_ACL_CHANGESETS_URL = "api/solr/aclchangesets"; private static final String GET_ACLS = "api/solr/acls"; private static final String GET_ACLS_READERS = "api/solr/aclsReaders"; @@ -142,7 +145,7 @@ public class SOLRAPIClient this.jsonFactory = new JsonFactory(); this.compression = compression; } - + /** * Get the ACL ChangeSets * @@ -179,32 +182,7 @@ public class SOLRAPIClient url.append(args); GetRequest req = new GetRequest(url.toString()); - Response response = null; - JSONObject json = null; - try - { - response = repositoryHttpClient.sendRequest(req); - - if (response.getStatus() != HttpStatus.SC_OK) - { - throw new AlfrescoRuntimeException(GET_ACL_CHANGESETS_URL + " return status:" + response.getStatus()); - } - - Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); - json = new JSONObject(new JSONTokener(reader)); - } - finally - { - if(response != null) - { - response.release(); - } - } - - if (log.isDebugEnabled()) - { - log.debug(json.toString(3)); - } + JSONObject json = callRepository(GET_ACL_CHANGESETS_URL, req); JSONArray aclChangeSetsJSON = json.getJSONArray("aclChangeSets"); List aclChangeSets = new ArrayList(aclChangeSetsJSON.length()); @@ -217,8 +195,7 @@ public class SOLRAPIClient AclChangeSet aclChangeSet = new AclChangeSet(aclChangeSetId, commitTimeMs, aclCount); aclChangeSets.add(aclChangeSet); } - - + Long maxChangeSetCommitTime = null; if(json.has("maxChangeSetCommitTime")) { @@ -273,32 +250,7 @@ public class SOLRAPIClient jsonReq.put("aclChangeSetIds", aclChangeSetIdsJSON); PostRequest req = new PostRequest(url.toString(), jsonReq.toString(), "application/json"); - Response response = null; - JSONObject json = null; - try - { - response = repositoryHttpClient.sendRequest(req); - - if (response.getStatus() != HttpStatus.SC_OK) - { - throw new AlfrescoRuntimeException(GET_ACL_CHANGESETS_URL + " return status:" + response.getStatus()); - } - - Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); - json = new JSONObject(new JSONTokener(reader)); - } - finally - { - if(response != null) - { - response.release(); - } - } - - if (log.isDebugEnabled()) - { - log.debug(json.toString(3)); - } + JSONObject json = callRepository(GET_ACL_CHANGESETS_URL, req); JSONArray aclsJSON = json.getJSONArray("acls"); List acls = new ArrayList(aclsJSON.length()); @@ -336,32 +288,7 @@ public class SOLRAPIClient jsonReq.put("aclIds", aclIdsJSON); PostRequest req = new PostRequest(url.toString(), jsonReq.toString(), "application/json"); - Response response = null; - JSONObject json = null; - try - { - response = repositoryHttpClient.sendRequest(req); - - if (response.getStatus() != HttpStatus.SC_OK) - { - throw new AlfrescoRuntimeException(GET_ACLS_READERS + " return status:" + response.getStatus()); - } - - Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); - json = new JSONObject(new JSONTokener(reader)); - } - finally - { - if(response != null) - { - response.release(); - } - } - - if (log.isDebugEnabled()) - { - log.debug(json.toString(3)); - } + JSONObject json = callRepository(GET_ACLS_READERS, req); JSONArray aclsReadersJSON = json.getJSONArray("aclsReaders"); List aclsReaders = new ArrayList(aclsReadersJSON.length()); @@ -421,7 +348,6 @@ public class SOLRAPIClient public Transactions getTransactions(Long fromCommitTime, Long minTxnId, Long toCommitTime, Long maxTxnId, int maxResults, ShardState shardState) throws AuthenticationException, IOException, JSONException, EncoderException { - log.debug("### get transactions ###"); URLCodec encoder = new URLCodec(); StringBuilder url = new StringBuilder(GET_TRANSACTIONS_URL); @@ -448,7 +374,7 @@ public class SOLRAPIClient } if(shardState != null) { - log.debug("### Shard state exists ###"); + LOGGER.debug("### Shard state exists ###"); args.append(args.length() == 0 ? "?" : "&"); args.append(encoder.encode("baseUrl")).append("=").append(encoder.encode(shardState.getShardInstance().getBaseUrl())); args.append("&").append(encoder.encode("hostName")).append("=").append(encoder.encode(shardState.getShardInstance().getHostName())); @@ -497,12 +423,14 @@ public class SOLRAPIClient } url.append(args); - log.debug("### GetRequest: " + url.toString()); + LOGGER.debug("### GetRequest: " + url.toString()); GetRequest req = new GetRequest(url.toString()); Response response = null; List transactions = new ArrayList(); Long maxTxnCommitTime = null; Long maxTxnIdOnServer = null; + + LookAheadBufferedReader reader = null; try { response = repositoryHttpClient.sendRequest(req); @@ -510,7 +438,8 @@ public class SOLRAPIClient { throw new AlfrescoRuntimeException("GetTransactions return status is " + response.getStatus()); } - Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); + + reader = new LookAheadBufferedReader(new InputStreamReader(response.getContentAsStream(), StandardCharsets.UTF_8), LOGGER); JsonParser parser = jsonFactory.createParser(reader); JsonToken token = parser.nextValue(); @@ -556,18 +485,25 @@ public class SOLRAPIClient token = parser.nextValue(); } parser.close(); - reader.close(); - + } + catch (JSONException exception) + { + String message = "Received a malformed JSON payload. Request was \"" + + req.getFullUri() + + "Data: " + + ofNullable(reader) + .map(LookAheadBufferedReader::lookAheadAndGetBufferedContent) + .orElse("Not available"); + LOGGER.error(message); + throw exception; } finally { - log.debug("## end getTransactions"); - if(response != null) - { - response.release(); - } + ofNullable(response).ifPresent(Response::release); + silentlyClose(reader); } - log.debug("### Transactions found maxTxnCommitTime: " + maxTxnCommitTime ); + + LOGGER.debug("### Transactions found maxTxnCommitTime: " + maxTxnCommitTime ); return new Transactions(transactions, maxTxnCommitTime, maxTxnIdOnServer); } @@ -637,35 +573,10 @@ public class SOLRAPIClient PostRequest req = new PostRequest(url.toString(), body.toString(), "application/json"); - - Response response = null; - JSONObject json = null; - try - { - response = repositoryHttpClient.sendRequest(req); - if(response.getStatus() != HttpStatus.SC_OK) - { - throw new AlfrescoRuntimeException("GetNodes return status is " + response.getStatus()); - } - - Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); - json = new JSONObject(new JSONTokener(reader)); - } - finally - { - if(response != null) - { - response.release(); - } - } - - if(log.isDebugEnabled()) - { - log.debug(json.toString()); - } + JSONObject json = callRepository(GET_NODES_URL, req); JSONArray jsonNodes = json.getJSONArray("nodes"); - List nodes = new ArrayList(jsonNodes.length()); + List nodes = new ArrayList<>(jsonNodes.length()); for(int i = 0; i < jsonNodes.length(); i++) { JSONObject jsonNodeInfo = jsonNodes.getJSONObject(i); @@ -893,34 +804,9 @@ public class SOLRAPIClient body.put("maxResults", maxResults); PostRequest req = new PostRequest(url.toString(), body.toString(), "application/json"); - Response response = null; - JSONObject json = null; - try - { - response = repositoryHttpClient.sendRequest(req); - if(response.getStatus() != HttpStatus.SC_OK) - { - throw new AlfrescoRuntimeException("GetNodeMetaData return status is " + response.getStatus()); - } - - Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); - json = new JSONObject(new JSONTokener(reader)); - } - finally - { - if(response != null) - { - response.release(); - } - } + JSONObject json = callRepository(GET_METADATA_URL, req); - if (log.isDebugEnabled()) - { - log.debug(json.toString(3)); - } - JSONArray jsonNodes = json.getJSONArray("nodes"); - List nodes = new ArrayList(jsonNodes.length()); for(int i = 0; i < jsonNodes.length(); i++) { @@ -1159,7 +1045,7 @@ public class SOLRAPIClient return new GetTextContentResponse(response); } - public AlfrescoModel getModel(String coreName, QName modelName) throws AuthenticationException, IOException, JSONException + public AlfrescoModel getModel(String coreName, QName modelName) throws AuthenticationException, IOException { // If the model is new to the SOLR side the prefix will be unknown so we can not generate prefixes for the request! // Always use the full QName with explicit URI @@ -1209,31 +1095,8 @@ public class SOLRAPIClient body.put("models", jsonModels); PostRequest req = new PostRequest(url.toString(), body.toString(), "application/json"); - Response response = null; - JSONObject json = null; - try - { - response = repositoryHttpClient.sendRequest(req); - if(response.getStatus() != HttpStatus.SC_OK) - { - throw new AlfrescoRuntimeException(coreName + " GetModelsDiff return status is " + response.getStatus()); - } - - Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); - json = new JSONObject(new JSONTokener(reader)); - } - finally - { - if(response != null) - { - response.release(); - } - } - - if(log.isDebugEnabled()) - { - log.debug(json.toString()); - } + JSONObject json = callRepository(GET_MODELS_DIFF, req); + JSONArray jsonDiffs = json.getJSONArray("diffs"); if(jsonDiffs == null) { @@ -1264,36 +1127,12 @@ public class SOLRAPIClient * @throws AuthenticationException * @throws NoSuchMethodException */ - public Long getNextTxCommitTime(String coreName, Long fromCommitTime) throws AuthenticationException, IOException, NoSuchMethodException + public Long getNextTxCommitTime(String coreName, Long fromCommitTime) throws AuthenticationException, IOException { StringBuilder url = new StringBuilder(GET_NEXT_TX_COMMIT_TIME); url.append("?").append("fromCommitTime").append("=").append(fromCommitTime); GetRequest get = new GetRequest(url.toString()); - Response response = null; - JSONObject json = null; - try - { - response = repositoryHttpClient.sendRequest(get); - if (response.getStatus() != HttpStatus.SC_OK) - { - throw new NoSuchMethodException(coreName + " - GetNextTxCommitTime return status is " - + response.getStatus() + " when invoking " + url); - } - - Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); - json = new JSONObject(new JSONTokener(reader)); - } - finally - { - if (response != null) - { - response.release(); - } - } - if (log.isDebugEnabled()) - { - log.debug(json.toString()); - } + JSONObject json = callRepository(GET_NEXT_TX_COMMIT_TIME, get); return Long.parseLong(json.get("nextTransactionCommitTimeMs").toString()); } @@ -1310,39 +1149,15 @@ public class SOLRAPIClient * @throws NoSuchMethodException */ public Pair getTxIntervalCommitTime(String coreName, Long fromNodeId, Long toNodeId) - throws AuthenticationException, IOException, NoSuchMethodException + throws AuthenticationException, IOException { StringBuilder url = new StringBuilder(GET_TX_INTERVAL_COMMIT_TIME); url.append("?").append("fromNodeId").append("=").append(fromNodeId); url.append("&").append("toNodeId").append("=").append(toNodeId); GetRequest get = new GetRequest(url.toString()); - Response response = null; - JSONObject json = null; - try - { - response = repositoryHttpClient.sendRequest(get); - if (response.getStatus() != HttpStatus.SC_OK) - { - throw new NoSuchMethodException(coreName + " - GetTxIntervalCommitTime return status is " - + response.getStatus() + " when invoking " + url); - } + JSONObject json = callRepository(GET_TX_INTERVAL_COMMIT_TIME, get); - Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); - json = new JSONObject(new JSONTokener(reader)); - } - finally - { - if (response != null) - { - response.release(); - } - } - if (log.isDebugEnabled()) - { - log.debug(json.toString()); - } - - return new Pair(Long.parseLong(json.get("minTransactionCommitTimeMs").toString()), + return new Pair<>(Long.parseLong(json.get("minTransactionCommitTimeMs").toString()), Long.parseLong(json.get("maxTransactionCommitTimeMs").toString())); } @@ -1692,4 +1507,56 @@ public class SOLRAPIClient { repositoryHttpClient.close(); } + + private JSONObject callRepository(String msgId, Request req) throws IOException, AuthenticationException + { + Response response = null; + LookAheadBufferedReader reader = null; + JSONObject json; + try + { + response = repositoryHttpClient.sendRequest(req); + if (response.getStatus() != HttpStatus.SC_OK) + { + throw new AlfrescoRuntimeException(msgId + " return status:" + response.getStatus()); + } + + reader = new LookAheadBufferedReader(new InputStreamReader(response.getContentAsStream(), StandardCharsets.UTF_8), LOGGER); + json = new JSONObject(new JSONTokener(reader)); + + if (LOGGER.isDebugEnabled()) + { + LOGGER.debug(json.toString(3)); + } + return json; + } + catch (JSONException exception) + { + String message = "Received a malformed JSON payload. Request was \"" + + req.getFullUri() + + "Data: " + + ofNullable(reader) + .map(LookAheadBufferedReader::lookAheadAndGetBufferedContent) + .orElse("Not available"); + LOGGER.error(message); + throw exception; + } + finally + { + ofNullable(response).ifPresent(Response::release); + ofNullable(reader).ifPresent(this::silentlyClose); + } + } + + private void silentlyClose(Closeable closeable) + { + try + { + closeable.close(); + } + catch (Exception ignore) + { + // Nothing to be done here + } + } } From 11e423ea84634cf39728ece846f74a806758878e Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 8 Apr 2020 21:50:45 +0200 Subject: [PATCH 02/54] [ SEARCH-2180 ] Fix compilation errors --- .../alfresco/solr/client/SOLRAPIClient.java | 80 +++++++++++++++++-- 1 file changed, 75 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index c64fd87a2..9f4b61de2 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -1127,12 +1127,47 @@ public class SOLRAPIClient * @throws AuthenticationException * @throws NoSuchMethodException */ - public Long getNextTxCommitTime(String coreName, Long fromCommitTime) throws AuthenticationException, IOException + public Long getNextTxCommitTime(String coreName, Long fromCommitTime) throws AuthenticationException, IOException, NoSuchMethodException { StringBuilder url = new StringBuilder(GET_NEXT_TX_COMMIT_TIME); url.append("?").append("fromCommitTime").append("=").append(fromCommitTime); GetRequest get = new GetRequest(url.toString()); - JSONObject json = callRepository(GET_NEXT_TX_COMMIT_TIME, get); + Response response = null; + JSONObject json = null; + LookAheadBufferedReader reader = null; + try + { + response = repositoryHttpClient.sendRequest(get); + if (response.getStatus() != HttpStatus.SC_OK) + { + throw new NoSuchMethodException(coreName + " - GetNextTxCommitTime return status is " + + response.getStatus() + " when invoking " + url); + } + + reader = new LookAheadBufferedReader(new InputStreamReader(response.getContentAsStream(), StandardCharsets.UTF_8), LOGGER); + json = new JSONObject(new JSONTokener(reader)); + } + catch (JSONException exception) + { + String message = "Received a malformed JSON payload. Request was \"" + + get.getFullUri() + + "Data: " + + ofNullable(reader) + .map(LookAheadBufferedReader::lookAheadAndGetBufferedContent) + .orElse("Not available"); + LOGGER.error(message); + throw exception; + } + finally + { + ofNullable(response).ifPresent(Response::release); + silentlyClose(reader); + } + + if (LOGGER.isDebugEnabled()) + { + LOGGER.debug(json.toString()); + } return Long.parseLong(json.get("nextTransactionCommitTimeMs").toString()); } @@ -1149,15 +1184,50 @@ public class SOLRAPIClient * @throws NoSuchMethodException */ public Pair getTxIntervalCommitTime(String coreName, Long fromNodeId, Long toNodeId) - throws AuthenticationException, IOException + throws AuthenticationException, IOException, NoSuchMethodException { StringBuilder url = new StringBuilder(GET_TX_INTERVAL_COMMIT_TIME); url.append("?").append("fromNodeId").append("=").append(fromNodeId); url.append("&").append("toNodeId").append("=").append(toNodeId); GetRequest get = new GetRequest(url.toString()); - JSONObject json = callRepository(GET_TX_INTERVAL_COMMIT_TIME, get); + Response response = null; + JSONObject json = null; + LookAheadBufferedReader reader = null; + try + { + response = repositoryHttpClient.sendRequest(get); + if (response.getStatus() != HttpStatus.SC_OK) + { + throw new NoSuchMethodException(coreName + " - GetTxIntervalCommitTime return status is " + + response.getStatus() + " when invoking " + url); + } - return new Pair<>(Long.parseLong(json.get("minTransactionCommitTimeMs").toString()), + reader = new LookAheadBufferedReader(new InputStreamReader(response.getContentAsStream(), StandardCharsets.UTF_8), LOGGER); + json = new JSONObject(new JSONTokener(reader)); + } + catch(JSONException exception) + { + String message = "Received a malformed JSON payload. Request was \"" + + get.getFullUri() + + "Data: " + + ofNullable(reader) + .map(LookAheadBufferedReader::lookAheadAndGetBufferedContent) + .orElse("Not available"); + LOGGER.error(message); + throw exception; + } + finally + { + ofNullable(response).ifPresent(Response::release); + silentlyClose(reader); + } + + if (LOGGER.isDebugEnabled()) + { + LOGGER.debug(json.toString()); + } + + return new Pair(Long.parseLong(json.get("minTransactionCommitTimeMs").toString()), Long.parseLong(json.get("maxTransactionCommitTimeMs").toString())); } From 9d8ee71e35a6a1073407381502a90a868b254e07 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Thu, 9 Apr 2020 10:22:46 +0200 Subject: [PATCH 03/54] [ SEARCH-2180 ] LookAheadBufferedReader UnitTest Case --- .../solr/client/LookAheadBufferedReader.java | 44 +++- .../client/LookAheadBufferedReaderTest.java | 210 ++++++++++++++++++ 2 files changed, 244 insertions(+), 10 deletions(-) create mode 100644 search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/LookAheadBufferedReaderTest.java diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java index 263074f50..786dd284c 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java @@ -42,6 +42,8 @@ import java.io.Reader; */ public class LookAheadBufferedReader extends BufferedReader { + final static String BUFFERING_DISABLED_INFO_MESSAGE = "Not available: please set the logging LEVEL to DEBUG or TRACE."; + private interface BufferingMode { void append(char ch); @@ -148,25 +150,20 @@ public class LookAheadBufferedReader extends BufferedReader @Override public String toString() { - return "Not available: please set the logging LEVEL to DEBUG or TRACE."; + return BUFFERING_DISABLED_INFO_MESSAGE; } } private final BufferingMode bufferingMode; - public LookAheadBufferedReader(Reader in, Logger logger) - { - this(in, 250, logger); - } - - public LookAheadBufferedReader(Reader in, final int windowSize, Logger logger) + LookAheadBufferedReader(Reader in, final int windowSize, boolean isDebugEnabled, boolean isTraceEnabled) { super(in); - if (logger.isTraceEnabled()) + if (isTraceEnabled) { bufferingMode = new WholeValue(); } - else if(logger.isDebugEnabled()) + else if(isDebugEnabled) { bufferingMode = new Windowing(windowSize); } @@ -175,11 +172,23 @@ public class LookAheadBufferedReader extends BufferedReader } } + public LookAheadBufferedReader(Reader in, final int windowSize, Logger logger) + { + this(in, windowSize, logger.isDebugEnabled(), logger.isTraceEnabled()); + } + + public LookAheadBufferedReader(Reader in, Logger logger) + { + this(in, 250, logger); + } + @Override public int read() throws IOException { int ch = super.read(); - bufferingMode.append((char)ch); + + if (ch != -1) bufferingMode.append((char)ch); + return ch; } @@ -200,4 +209,19 @@ public class LookAheadBufferedReader extends BufferedReader } return bufferingMode.toString(); } + + boolean isInWindowingMode() + { + return bufferingMode instanceof Windowing; + } + + boolean isInCollectEverythingMode() + { + return bufferingMode instanceof WholeValue; + } + + boolean isBufferingDisabled() + { + return bufferingMode instanceof NoOp; + } } \ No newline at end of file diff --git a/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/LookAheadBufferedReaderTest.java b/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/LookAheadBufferedReaderTest.java new file mode 100644 index 000000000..050f91bbc --- /dev/null +++ b/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/LookAheadBufferedReaderTest.java @@ -0,0 +1,210 @@ +/*- + * #%L + * Alfresco Remote API + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.solr.client; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.Reader; +import java.io.StringReader; + +import static java.util.stream.IntStream.range; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class LookAheadBufferedReaderTest +{ + @Mock + Reader reader; + + private final String data = "1234567890ABCDEFGHILMNOPQRSTUVYXZ"; + + @Test + public void windowingModeEnabled() + { + LookAheadBufferedReader classUnderTest = new LookAheadBufferedReader(reader, data.length(), true, false); + assertTrue(classUnderTest.isInWindowingMode()); + assertFalse(classUnderTest.isInCollectEverythingMode()); + assertFalse(classUnderTest.isBufferingDisabled()); + } + + @Test + public void collectEverythingModeEnabled() + { + LookAheadBufferedReader classUnderTest = new LookAheadBufferedReader(reader, data.length(), false, true); + assertTrue(classUnderTest.isInCollectEverythingMode()); + assertFalse(classUnderTest.isInWindowingMode()); + assertFalse(classUnderTest.isBufferingDisabled()); + } + + @Test + public void bufferingDisabled() + { + LookAheadBufferedReader classUnderTest = new LookAheadBufferedReader(reader, data.length(), false, false); + assertTrue(classUnderTest.isBufferingDisabled()); + assertFalse(classUnderTest.isInCollectEverythingMode()); + assertFalse(classUnderTest.isInWindowingMode()); + } + + @Test + public void collectEverythingWinsOverWindowing() + { + LookAheadBufferedReader classUnderTest = new LookAheadBufferedReader(reader, data.length(), true, true); + assertTrue(classUnderTest.isInCollectEverythingMode()); + assertFalse(classUnderTest.isInWindowingMode()); + assertFalse(classUnderTest.isBufferingDisabled()); + } + + @Test + public void windowingModeShouldCollectPartialWindowsOfData() + { + int windowSize = 10; + Reader reader = new StringReader(data); + LookAheadBufferedReader classUnderTest = new LookAheadBufferedReader(reader, windowSize, true, false); + + // Read only 12 chars from the underlying stream + range(0, 12).forEach(index -> consume(classUnderTest)); + + String collectedWindow = classUnderTest.lookAheadAndGetBufferedContent(); + + assertEquals(windowSize * 2, collectedWindow.length()); + assertEquals("4567890ABCDEFGHILMNO", collectedWindow); + } + + @Test + public void notEnoughCharsForTheWindow() + { + int windowSize = 10; + Reader reader = new StringReader(data); + LookAheadBufferedReader classUnderTest = new LookAheadBufferedReader(reader, windowSize, true, false); + + range(0, data.length() - 3).forEach(index -> consume(classUnderTest)); + + String collectedWindow = classUnderTest.lookAheadAndGetBufferedContent(); + + assertEquals(windowSize + 2, collectedWindow.length()); + assertEquals("NOPQRSTUVYXZ", collectedWindow); + } + + @Test + public void emptyDataShouldCollectEmptyWindow() + { + Reader reader = new StringReader(""); + LookAheadBufferedReader classUnderTest = new LookAheadBufferedReader(reader, 10, true, false); + + range(0, data.length() - 3).forEach(index -> consume(classUnderTest)); + + String collectedWindow = classUnderTest.lookAheadAndGetBufferedContent(); + + assertEquals(0, collectedWindow.length()); + assertEquals("", collectedWindow); + } + + @Test + public void collectEverythingModeShouldCollectTheWholeStream() + { + Reader reader = new StringReader(data); + LookAheadBufferedReader classUnderTest = + new LookAheadBufferedReader( + reader, + 10, // this has no effect + false, + true); + + // Read only 12 chars from the underlying stream + range(0, 12).forEach(index -> consume(classUnderTest)); + + String collectedData = classUnderTest.lookAheadAndGetBufferedContent(); + + assertEquals(data.length(), collectedData.length()); + assertEquals(data, collectedData); + } + + @Test + public void emptyDataShouldCollectEmptyStringInCollectEverythingMode() + { + Reader reader = new StringReader(""); + LookAheadBufferedReader classUnderTest = + new LookAheadBufferedReader( + reader, + 10, // this has no effect + false, + true); + + range(0, data.length() - 3).forEach(index -> consume(classUnderTest)); + + String collectedData = classUnderTest.lookAheadAndGetBufferedContent(); + + assertEquals(0, collectedData.length()); + assertEquals("", collectedData); + } + + @Test + public void bufferingModeDisableShouldCollectNoData() + { + int windowSize = 10; + Reader reader = new StringReader(data); + LookAheadBufferedReader classUnderTest = new LookAheadBufferedReader(reader, windowSize, false, false); + + // Read only 12 chars from the underlying stream + range(0, 12).forEach(index -> consume(classUnderTest)); + assertEquals(LookAheadBufferedReader.BUFFERING_DISABLED_INFO_MESSAGE, classUnderTest.lookAheadAndGetBufferedContent()); + } + + @Test + public void emptyDataShouldCollectEmptyDataWhenBufferingIsDisabled() + { + Reader reader = new StringReader(""); + LookAheadBufferedReader classUnderTest = + new LookAheadBufferedReader( + reader, + 10, // this has no effect + false, + false); + + range(0, data.length() - 3).forEach(index -> consume(classUnderTest)); + + String collectedData = classUnderTest.lookAheadAndGetBufferedContent(); + + assertEquals(LookAheadBufferedReader.BUFFERING_DISABLED_INFO_MESSAGE, collectedData); + } + + private void consume(Reader reader) + { + try + { + reader.read(); + } + catch (Exception exception) + { + throw new RuntimeException(exception); + } + } +} \ No newline at end of file From bf64a796ca3d1efffde84534bf3e71bd0877a86b Mon Sep 17 00:00:00 2001 From: agazzarini Date: Thu, 9 Apr 2020 12:03:42 +0200 Subject: [PATCH 04/54] [ SEARCH-2180 ] Review comments addressed --- .../solr/client/LookAheadBufferedReader.java | 18 ++++++++++++++++++ .../alfresco/solr/client/SOLRAPIClient.java | 6 +++--- .../client/LookAheadBufferedReaderTest.java | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java index 786dd284c..50edf4018 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/LookAheadBufferedReader.java @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2005-2020 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ package org.alfresco.solr.client; import java.io.BufferedReader; diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 9f4b61de2..628114700 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -500,7 +500,7 @@ public class SOLRAPIClient finally { ofNullable(response).ifPresent(Response::release); - silentlyClose(reader); + ofNullable(reader).ifPresent(this::silentlyClose); } LOGGER.debug("### Transactions found maxTxnCommitTime: " + maxTxnCommitTime ); @@ -1161,7 +1161,7 @@ public class SOLRAPIClient finally { ofNullable(response).ifPresent(Response::release); - silentlyClose(reader); + ofNullable(reader).ifPresent(this::silentlyClose); } if (LOGGER.isDebugEnabled()) @@ -1219,7 +1219,7 @@ public class SOLRAPIClient finally { ofNullable(response).ifPresent(Response::release); - silentlyClose(reader); + ofNullable(reader).ifPresent(this::silentlyClose); } if (LOGGER.isDebugEnabled()) diff --git a/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/LookAheadBufferedReaderTest.java b/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/LookAheadBufferedReaderTest.java index 050f91bbc..f51142c43 100644 --- a/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/LookAheadBufferedReaderTest.java +++ b/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/LookAheadBufferedReaderTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Remote API * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited + * Copyright (C) 2005 - 2020 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of From c639ffcf9b000a34e5affc08867961b8328d4311 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Fri, 10 Apr 2020 08:07:11 +0200 Subject: [PATCH 05/54] [ SEARCH-2181 ] RepositoryClient exceptions handling in SolrInformationServer --- .../alfresco/solr/SolrInformationServer.java | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 491f2ef0a..7b70144db 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 Alfresco Software Limited. + * Copyright (C) 2020 Alfresco Software Limited. * * This file is part of Alfresco * @@ -69,6 +69,7 @@ import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TXCOM import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TXID; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TYPE; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_VERSION; +import static org.alfresco.solr.utils.Utils.notNullOrEmpty; import java.io.File; import java.io.IOException; @@ -85,7 +86,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -144,6 +144,7 @@ import org.alfresco.solr.content.SolrContentStore; import org.alfresco.solr.logging.Log; import org.alfresco.solr.tracker.IndexHealthReport; import org.alfresco.solr.tracker.TrackerStats; +import org.alfresco.solr.utils.Utils; import org.alfresco.util.ISO9075; import org.alfresco.util.Pair; import org.apache.commons.io.input.BoundedInputStream; @@ -451,7 +452,7 @@ public class SolrInformationServer implements InformationServer this.core = core; this.nativeRequestHandler = core.getRequestHandler(REQUEST_HANDLER_NATIVE); this.cloud = new Cloud(); - this.repositoryClient = repositoryClient; + this.repositoryClient = Objects.requireNonNull(repositoryClient); this.solrContentStore = solrContentStore; Properties p = core.getResourceLoader().getCoreProperties(); @@ -1502,7 +1503,7 @@ public class SolrInformationServer implements InformationServer NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); nmdp.setFromNodeId(node.getId()); nmdp.setToNodeId(node.getId()); - List nodeMetaDatas; + Collection nodeMetaDatas; if ((node.getStatus() == SolrApiNodeStatus.DELETED) || cascadeTrackingEnabled() && ((node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED))) @@ -1513,13 +1514,13 @@ public class SolrInformationServer implements InformationServer } else { - nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE); + nodeMetaDatas = nodesMetaDataFromRepository(nmdp, Integer.MAX_VALUE); } NodeMetaData nodeMetaData; if (!nodeMetaDatas.isEmpty()) { - nodeMetaData = nodeMetaDatas.get(0); + nodeMetaData = nodeMetaDatas.iterator().next(); if (!(nodeMetaData.getTxnId() > node.getTxnId())) { if (node.getStatus() == SolrApiNodeStatus.DELETED) @@ -1558,14 +1559,14 @@ public class SolrInformationServer implements InformationServer nmdp.setFromNodeId(node.getId()); nmdp.setToNodeId(node.getId()); - List nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE); + Collection nodeMetaDatas = nodesMetaDataFromRepository(nmdp, Integer.MAX_VALUE); AddUpdateCommand addDocCmd = new AddUpdateCommand(request); addDocCmd.overwrite = overwrite; if (!nodeMetaDatas.isEmpty()) { - NodeMetaData nodeMetaData = nodeMetaDatas.get(0); + NodeMetaData nodeMetaData = nodeMetaDatas.iterator().next(); if(node.getTxnId() == Long.MAX_VALUE) { //This is a re-index. We need to clear the txnId from the pr this.cleanContentCache.remove(nodeMetaData.getTxnId()); @@ -1723,7 +1724,7 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludeProperties(false); nmdp.setIncludeTxnId(true); // Gets only one - List nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, 1); + Collection nodeMetaDatas = nodesMetaDataFromRepository(nmdp, 1); allNodeMetaDatas.addAll(nodeMetaDatas); } @@ -1808,6 +1809,10 @@ public class SolrInformationServer implements InformationServer processor.processAdd(addDocCmd); } } + catch (Exception exception) + { + LOGGER.error("Unable to update the text content of node {}. See the stacktrace below for further details.", dbId, exception); + } finally { unlock(dbId); @@ -1862,7 +1867,7 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludeAspects(false); nmdp.setIncludePaths(false); nmdp.setIncludeParentAssociations(false); - nodeMetaDatas.addAll(repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE)); + nodeMetaDatas.addAll(nodesMetaDataFromRepository(nmdp, Integer.MAX_VALUE)); } for (NodeMetaData nodeMetaData : nodeMetaDatas) @@ -1907,7 +1912,7 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludeChildAssociations(false); // Fetches bulk metadata - List nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE); + Collection nodeMetaDatas = nodesMetaDataFromRepository(nmdp, Integer.MAX_VALUE); NEXT_NODE: for (NodeMetaData nodeMetaData : nodeMetaDatas) @@ -2484,11 +2489,11 @@ public class SolrInformationServer implements InformationServer NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); nmdp.setFromNodeId(dbId); nmdp.setToNodeId(dbId); - List nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE); + Collection nodeMetaDatas = nodesMetaDataFromRepository(nmdp, Integer.MAX_VALUE); SolrInputDocument newDoc = null; if (!nodeMetaDatas.isEmpty()) { - NodeMetaData nodeMetaData = nodeMetaDatas.get(0); + NodeMetaData nodeMetaData = nodeMetaDatas.iterator().next(); newDoc = createNewDoc(nodeMetaData, DOC_TYPE_NODE); addFieldsToDoc(nodeMetaData, newDoc); boolean isContentIndexedForNode = isContentIndexedForNode(nodeMetaData.getProperties()); @@ -3376,11 +3381,11 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludeType(false); nmdp.setIncludeTxnId(true); // Gets only one - List nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, 1); + Collection nodeMetaDatas = nodesMetaDataFromRepository(nmdp, 1); if (!nodeMetaDatas.isEmpty()) { - NodeMetaData nodeMetaData = nodeMetaDatas.get(0); + NodeMetaData nodeMetaData = nodeMetaDatas.iterator().next(); // Only cascade update nods we know can not have changed and must be in this shard // Node in the current TX will be explicitly updated in the outer loop @@ -3823,4 +3828,22 @@ public class SolrInformationServer implements InformationServer { solrContentStore.flushChangeSet(); } + + private Collection nodesMetaDataFromRepository(NodeMetaDataParameters parameters, int maxResults) + { + try + { + return notNullOrEmpty(repositoryClient.getNodesMetaData(parameters, maxResults)); + } + catch (JSONException exception) + { + // Nothing to be done here: the exception has been already logged in repositoryClient + return Collections.emptyList(); + } + catch (Exception exception) + { + LOGGER.error("Unable to get nodes metadata from repository. See the stacktrace below for further details."); + return Collections.emptyList(); + } + } } \ No newline at end of file From 05338ac8a12e2a852c9d3c74283bac5329454821 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 10 Apr 2020 21:13:18 +0000 Subject: [PATCH 06/54] Bump restapi from 1.37 to 1.38 in /e2e-test Bumps [restapi](https://github.com/Alfresco/alfresco-tas-restapi) from 1.37 to 1.38. - [Release notes](https://github.com/Alfresco/alfresco-tas-restapi/releases) - [Commits](https://github.com/Alfresco/alfresco-tas-restapi/compare/v1.37...v1.38) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 3170f8cfb..c7ea6717b 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,7 +11,7 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 1.37 + 1.38 1.13 3.0.20 3.3.0.1 From 773cb4b5135de76b83e166b6de5cbf2e4140bace Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2020 21:13:22 +0000 Subject: [PATCH 07/54] Bump alfresco-data-model from 8.98 to 8.102 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.98 to 8.102. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.98...8.102) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 6ab4eda5a..17476b908 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.98 + 8.102 2.10.3 From 4b901e5846a22e1a77c7920296d0bd884242efb5 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Tue, 14 Apr 2020 14:23:34 +0200 Subject: [PATCH 08/54] [ SEARCH-2181 ] Use MaxResults in NodeMetaDataParameters --- .../alfresco/solr/SolrInformationServer.java | 29 ++++++---- .../solr/client/NodeMetaDataParameters.java | 57 ++++++++++--------- .../alfresco/solr/client/SOLRAPIClient.java | 9 ++- 3 files changed, 52 insertions(+), 43 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 7b70144db..7191588a6 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -144,7 +144,6 @@ import org.alfresco.solr.content.SolrContentStore; import org.alfresco.solr.logging.Log; import org.alfresco.solr.tracker.IndexHealthReport; import org.alfresco.solr.tracker.TrackerStats; -import org.alfresco.solr.utils.Utils; import org.alfresco.util.ISO9075; import org.alfresco.util.Pair; import org.apache.commons.io.input.BoundedInputStream; @@ -1514,7 +1513,8 @@ public class SolrInformationServer implements InformationServer } else { - nodeMetaDatas = nodesMetaDataFromRepository(nmdp, Integer.MAX_VALUE); + nmdp.setMaxResults(Integer.MAX_VALUE); + nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); } NodeMetaData nodeMetaData; @@ -1558,8 +1558,8 @@ public class SolrInformationServer implements InformationServer NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); nmdp.setFromNodeId(node.getId()); nmdp.setToNodeId(node.getId()); - - Collection nodeMetaDatas = nodesMetaDataFromRepository(nmdp, Integer.MAX_VALUE); + nmdp.setMaxResults(Integer.MAX_VALUE); + Collection nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); AddUpdateCommand addDocCmd = new AddUpdateCommand(request); addDocCmd.overwrite = overwrite; @@ -1723,8 +1723,9 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludePaths(true); nmdp.setIncludeProperties(false); nmdp.setIncludeTxnId(true); + nmdp.setMaxResults(1); // Gets only one - Collection nodeMetaDatas = nodesMetaDataFromRepository(nmdp, 1); + Collection nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); allNodeMetaDatas.addAll(nodeMetaDatas); } @@ -1867,7 +1868,8 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludeAspects(false); nmdp.setIncludePaths(false); nmdp.setIncludeParentAssociations(false); - nodeMetaDatas.addAll(nodesMetaDataFromRepository(nmdp, Integer.MAX_VALUE)); + nmdp.setMaxResults(Integer.MAX_VALUE); + nodeMetaDatas.addAll(getNodesMetaDataFromRepository(nmdp)); } for (NodeMetaData nodeMetaData : nodeMetaDatas) @@ -1912,7 +1914,8 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludeChildAssociations(false); // Fetches bulk metadata - Collection nodeMetaDatas = nodesMetaDataFromRepository(nmdp, Integer.MAX_VALUE); + nmdp.setMaxResults(Integer.MAX_VALUE); + Collection nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); NEXT_NODE: for (NodeMetaData nodeMetaData : nodeMetaDatas) @@ -2489,7 +2492,8 @@ public class SolrInformationServer implements InformationServer NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); nmdp.setFromNodeId(dbId); nmdp.setToNodeId(dbId); - Collection nodeMetaDatas = nodesMetaDataFromRepository(nmdp, Integer.MAX_VALUE); + nmdp.setMaxResults(Integer.MAX_VALUE); + Collection nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); SolrInputDocument newDoc = null; if (!nodeMetaDatas.isEmpty()) { @@ -3380,8 +3384,9 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludeProperties(false); nmdp.setIncludeType(false); nmdp.setIncludeTxnId(true); + nmdp.setMaxResults(1); // Gets only one - Collection nodeMetaDatas = nodesMetaDataFromRepository(nmdp, 1); + Collection nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); if (!nodeMetaDatas.isEmpty()) { @@ -3829,11 +3834,11 @@ public class SolrInformationServer implements InformationServer solrContentStore.flushChangeSet(); } - private Collection nodesMetaDataFromRepository(NodeMetaDataParameters parameters, int maxResults) + private Collection getNodesMetaDataFromRepository(NodeMetaDataParameters parameters) { try { - return notNullOrEmpty(repositoryClient.getNodesMetaData(parameters, maxResults)); + return notNullOrEmpty(repositoryClient.getNodesMetaData(parameters)); } catch (JSONException exception) { @@ -3842,7 +3847,7 @@ public class SolrInformationServer implements InformationServer } catch (Exception exception) { - LOGGER.error("Unable to get nodes metadata from repository. See the stacktrace below for further details."); + LOGGER.error("Unable to get nodes metadata from repository. See the stacktrace below for further details.", exception); return Collections.emptyList(); } } diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/NodeMetaDataParameters.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/NodeMetaDataParameters.java index 20f67ceef..303a32af1 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/NodeMetaDataParameters.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/NodeMetaDataParameters.java @@ -1,31 +1,32 @@ -/* - * #%L - * Alfresco Solr Client - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Solr Client + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.solr.client; import java.util.List; +import java.util.OptionalInt; /** * Stores node meta data query parameters for use in SOLR remote api calls @@ -39,7 +40,7 @@ public class NodeMetaDataParameters private Long toTxnId; // default is 'all' results - private int maxResults = 0; + private OptionalInt maxResults = OptionalInt.empty(); private Long fromNodeId; private Long toNodeId; @@ -147,14 +148,14 @@ public class NodeMetaDataParameters this.includeParentAssociations = includeParentAssociations; } - public int getMaxResults() + public OptionalInt getMaxResults() { return maxResults; } public void setMaxResults(int maxResults) { - this.maxResults = maxResults; + this.maxResults = OptionalInt.of(maxResults); } public List getNodeIds() diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 628114700..51864069b 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -732,7 +732,7 @@ public class SOLRAPIClient return ret; } - public List getNodesMetaData(NodeMetaDataParameters params, int maxResults) throws AuthenticationException, IOException, JSONException + public List getNodesMetaData(NodeMetaDataParameters params) throws AuthenticationException, IOException, JSONException { List nodeIds = params.getNodeIds(); @@ -801,13 +801,16 @@ public class SOLRAPIClient body.put("includeTxnId", params.isIncludeTxnId()); } - body.put("maxResults", maxResults); + if (params.getMaxResults().isPresent()) + { + body.put("maxResults", params.getMaxResults()); + } PostRequest req = new PostRequest(url.toString(), body.toString(), "application/json"); JSONObject json = callRepository(GET_METADATA_URL, req); JSONArray jsonNodes = json.getJSONArray("nodes"); - List nodes = new ArrayList(jsonNodes.length()); + List nodes = new ArrayList<>(jsonNodes.length()); for(int i = 0; i < jsonNodes.length(); i++) { JSONObject jsonNodeInfo = jsonNodes.getJSONObject(i); From 15b4dacc4c90305396914416beae99be579c46a6 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Tue, 14 Apr 2020 14:29:38 +0200 Subject: [PATCH 09/54] [ SEARCH-2181 ] Use MaxResults in NodeMetaDataParameters (fix tests) --- .../solr/client/SOLRAPIClientTest.java | 224 +++++++++--------- 1 file changed, 116 insertions(+), 108 deletions(-) diff --git a/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/SOLRAPIClientTest.java b/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/SOLRAPIClientTest.java index 6836a0556..6e61c9c4a 100644 --- a/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/SOLRAPIClientTest.java +++ b/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/client/SOLRAPIClientTest.java @@ -1,93 +1,93 @@ -/* - * #%L - * Alfresco Solr Client - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Solr Client + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.solr.client; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.security.AlgorithmParameters; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.alfresco.encryption.DefaultEncryptionUtils; -import org.alfresco.encryption.KeyProvider; -import org.alfresco.encryption.KeyResourceLoader; -import org.alfresco.encryption.KeyStoreParameters; -import org.alfresco.encryption.MACUtils.MACInput; -import org.alfresco.encryption.ssl.SSLEncryptionParameters; -import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.httpclient.AlfrescoHttpClient; -import org.alfresco.httpclient.AuthenticationException; -import org.alfresco.httpclient.HttpClientFactory; -import org.alfresco.httpclient.HttpClientFactory.SecureCommsType; -import org.alfresco.opencmis.dictionary.CMISDictionaryRegistry; -import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService; -import org.alfresco.opencmis.mapping.CMISMapping; -import org.alfresco.opencmis.mapping.RuntimePropertyLuceneBuilderMapping; -import org.alfresco.repo.cache.MemoryCache; -import org.alfresco.repo.dictionary.CompiledModelsCache; -import org.alfresco.repo.dictionary.DictionaryComponent; -import org.alfresco.repo.dictionary.DictionaryDAOImpl; -import org.alfresco.repo.dictionary.DictionaryNamespaceComponent; -import org.alfresco.repo.dictionary.M2Model; -import org.alfresco.repo.dictionary.M2Namespace; -import org.alfresco.repo.dictionary.NamespaceDAO; -import org.alfresco.repo.i18n.StaticMessageLookup; -import org.alfresco.repo.tenant.SingleTServiceImpl; -import org.alfresco.repo.tenant.TenantService; -import org.alfresco.service.namespace.QName; -import org.alfresco.util.DynamicallySizedThreadPoolExecutor; -import org.alfresco.util.Pair; -import org.alfresco.util.TraceableThreadFactory; -import org.alfresco.util.cache.DefaultAsynchronouslyRefreshedCacheRegistry; -import org.apache.chemistry.opencmis.commons.enums.CmisVersion; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.methods.ByteArrayRequestEntity; -import org.apache.commons.httpclient.methods.PostMethod; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.json.JSONException; - -import junit.framework.TestCase; - +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.AlgorithmParameters; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.alfresco.encryption.DefaultEncryptionUtils; +import org.alfresco.encryption.KeyProvider; +import org.alfresco.encryption.KeyResourceLoader; +import org.alfresco.encryption.KeyStoreParameters; +import org.alfresco.encryption.MACUtils.MACInput; +import org.alfresco.encryption.ssl.SSLEncryptionParameters; +import org.alfresco.error.AlfrescoRuntimeException; +import org.alfresco.httpclient.AlfrescoHttpClient; +import org.alfresco.httpclient.AuthenticationException; +import org.alfresco.httpclient.HttpClientFactory; +import org.alfresco.httpclient.HttpClientFactory.SecureCommsType; +import org.alfresco.opencmis.dictionary.CMISDictionaryRegistry; +import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService; +import org.alfresco.opencmis.mapping.CMISMapping; +import org.alfresco.opencmis.mapping.RuntimePropertyLuceneBuilderMapping; +import org.alfresco.repo.cache.MemoryCache; +import org.alfresco.repo.dictionary.CompiledModelsCache; +import org.alfresco.repo.dictionary.DictionaryComponent; +import org.alfresco.repo.dictionary.DictionaryDAOImpl; +import org.alfresco.repo.dictionary.DictionaryNamespaceComponent; +import org.alfresco.repo.dictionary.M2Model; +import org.alfresco.repo.dictionary.M2Namespace; +import org.alfresco.repo.dictionary.NamespaceDAO; +import org.alfresco.repo.i18n.StaticMessageLookup; +import org.alfresco.repo.tenant.SingleTServiceImpl; +import org.alfresco.repo.tenant.TenantService; +import org.alfresco.service.namespace.QName; +import org.alfresco.util.DynamicallySizedThreadPoolExecutor; +import org.alfresco.util.Pair; +import org.alfresco.util.TraceableThreadFactory; +import org.alfresco.util.cache.DefaultAsynchronouslyRefreshedCacheRegistry; +import org.apache.chemistry.opencmis.commons.enums.CmisVersion; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.methods.ByteArrayRequestEntity; +import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.json.JSONException; + +import junit.framework.TestCase; + /** * Tests {@link SOLRAPIClient} Note: need to make sure that source/solr/instance is on the run classpath. Note: doesn't * currently work, need to change to use SSL. @@ -109,7 +109,7 @@ public class SOLRAPIClientTest extends TestCase private DictionaryDAOImpl dictionaryDAO; private CMISStrictDictionaryService cmisDictionaryService; - + private static final String CORENAME = "collection1"; // private M2Model testModel; @@ -140,10 +140,10 @@ public class SOLRAPIClientTest extends TestCase { if(client == null) { - TenantService tenantService = new SingleTServiceImpl(); + TenantService tenantService = new SingleTServiceImpl(); dictionaryDAO = new DictionaryDAOImpl(); - NamespaceDAO namespaceDAO = dictionaryDAO; + NamespaceDAO namespaceDAO = dictionaryDAO; dictionaryDAO.setTenantService(tenantService); CompiledModelsCache compiledModelsCache = new CompiledModelsCache(); @@ -163,7 +163,7 @@ public class SOLRAPIClientTest extends TestCase dictionaryDAO.setResourceClassLoader(getResourceClassLoader()); dictionaryDAO.init(); - DictionaryComponent dictionaryComponent = new DictionaryComponent(); + DictionaryComponent dictionaryComponent = new DictionaryComponent(); dictionaryComponent.setDictionaryDAO(dictionaryDAO); dictionaryComponent.setMessageLookup(new StaticMessageLookup()); @@ -283,7 +283,7 @@ public class SOLRAPIClientTest extends TestCase 8443, 40, 40, 0); // TODO need to make port configurable depending on secure comms, or just make redirects // work - return httpClientFactory.getRepoClient("localhost", 8443); + return httpClientFactory.getRepoClient("localhost", 8443); } public ClassLoader getResourceClassLoader() @@ -452,7 +452,8 @@ public class SOLRAPIClientTest extends TestCase NodeMetaDataParameters metaParams = new NodeMetaDataParameters(); metaParams.setNodeIds(nodeIds); - List metadata = client.getNodesMetaData(metaParams, 3); + metaParams.setMaxResults(3); + List metadata = client.getNodesMetaData(metaParams); for (NodeMetaData info : metadata) { logger.debug(info); @@ -463,9 +464,10 @@ public class SOLRAPIClientTest extends TestCase { NodeMetaDataParameters metaParams = new NodeMetaDataParameters(); List nodeIds = new ArrayList(1); - nodeIds.add(1l); + nodeIds.add(1L); + metaParams.setMaxResults(3); metaParams.setNodeIds(nodeIds); - List metadata = client.getNodesMetaData(metaParams, 3); + List metadata = client.getNodesMetaData(metaParams); for (NodeMetaData info : metadata) { logger.debug(info); @@ -473,9 +475,10 @@ public class SOLRAPIClientTest extends TestCase metaParams = new NodeMetaDataParameters(); nodeIds = new ArrayList(1); - nodeIds.add(9l); + nodeIds.add(9L); metaParams.setNodeIds(nodeIds); - metadata = client.getNodesMetaData(metaParams, 3); + metaParams.setMaxResults(3); + metadata = client.getNodesMetaData(metaParams); for (NodeMetaData info : metadata) { logger.debug(info); @@ -483,9 +486,10 @@ public class SOLRAPIClientTest extends TestCase metaParams = new NodeMetaDataParameters(); nodeIds = new ArrayList(1); - nodeIds.add(19l); + nodeIds.add(19L); metaParams.setNodeIds(nodeIds); - metadata = client.getNodesMetaData(metaParams, 3); + metaParams.setMaxResults(3); + metadata = client.getNodesMetaData(metaParams); for (NodeMetaData info : metadata) { logger.debug(info); @@ -495,9 +499,10 @@ public class SOLRAPIClientTest extends TestCase // TODO check why the category path has a null QName metaParams = new NodeMetaDataParameters(); nodeIds = new ArrayList(1); - nodeIds.add(49437l); + nodeIds.add(49437L); metaParams.setNodeIds(nodeIds); - metadata = client.getNodesMetaData(metaParams, 3); + metaParams.setMaxResults(3); + metadata = client.getNodesMetaData(metaParams); for (NodeMetaData info : metadata) { logger.debug(info); @@ -506,9 +511,10 @@ public class SOLRAPIClientTest extends TestCase // content with tags metaParams = new NodeMetaDataParameters(); nodeIds = new ArrayList(1); - nodeIds.add(49431l); + nodeIds.add(49431L); metaParams.setNodeIds(nodeIds); - metadata = client.getNodesMetaData(metaParams, 3); + metaParams.setMaxResults(3); + metadata = client.getNodesMetaData(metaParams); for (NodeMetaData info : metadata) { logger.debug(info); @@ -525,9 +531,10 @@ public class SOLRAPIClientTest extends TestCase // content with null property values for author metaParams = new NodeMetaDataParameters(); nodeIds = new ArrayList(1); - nodeIds.add(117630l); + nodeIds.add(117630L); metaParams.setNodeIds(nodeIds); - metadata = client.getNodesMetaData(metaParams, 3); + metaParams.setMaxResults(3); + metadata = client.getNodesMetaData(metaParams); for (NodeMetaData info : metadata) { logger.debug(info); @@ -536,9 +543,10 @@ public class SOLRAPIClientTest extends TestCase // content with accented characters in title properties metaParams = new NodeMetaDataParameters(); nodeIds = new ArrayList(1); - nodeIds.add(117678l); + nodeIds.add(117678L); metaParams.setNodeIds(nodeIds); - metadata = client.getNodesMetaData(metaParams, 3); + metaParams.setMaxResults(3); + metadata = client.getNodesMetaData(metaParams); for (NodeMetaData info : metadata) { logger.debug(info); From 3e6bee8b8f1ef1f9acecf36ddc24bfb0435f6773 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Tue, 14 Apr 2020 17:00:29 +0200 Subject: [PATCH 10/54] [ SEARCH-2181 ] AlfrescoLockException and generic exception catch in lock/unlock blocks --- .../alfresco/solr/AlfrescoLockException.java | 30 ++++++++++++++++ .../alfresco/solr/SolrInformationServer.java | 36 ++++++++++++++++--- 2 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoLockException.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoLockException.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoLockException.java new file mode 100644 index 000000000..d6a5d3c89 --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoLockException.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2005-2020 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr; + +/** + * Marker exception thrown for indicating a failure in obtaining / releasing a lock. + */ +public class AlfrescoLockException extends Exception +{ + AlfrescoLockException(String message) + { + super(message); + } +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 7191588a6..95a596224 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -1810,6 +1810,10 @@ public class SolrInformationServer implements InformationServer processor.processAdd(addDocCmd); } } + catch (AlfrescoLockException exception) + { + LOGGER.error(exception.getMessage()); + } catch (Exception exception) { LOGGER.error("Unable to update the text content of node {}. See the stacktrace below for further details.", dbId, exception); @@ -1888,6 +1892,14 @@ public class SolrInformationServer implements InformationServer solrContentStore.removeDocFromContentStore(nodeMetaData); } + catch (AlfrescoLockException exception) + { + LOGGER.error(exception.getMessage()); + } + catch (Exception exception) + { + LOGGER.error("Unable to remove document {} from the content store. See the stacktrace below for further details.", nodeMetaData.getId(), exception); + } finally { unlock(nodeMetaData.getId()); @@ -1985,6 +1997,14 @@ public class SolrInformationServer implements InformationServer long end = System.nanoTime(); this.trackerStats.addNodeTime(end - start); } + catch (AlfrescoLockException exception) + { + LOGGER.error(exception.getMessage()); + } + catch (Exception exception) + { + LOGGER.error("Upsert failure on Node {}. See the stacktrace below for further details.", nodeId, exception); + } finally { unlock(nodeId); @@ -3164,10 +3184,10 @@ public class SolrInformationServer implements InformationServer } } - private void lock(Object id) throws IOException + private void lock(Object id) throws AlfrescoLockException { long startTime = System.currentTimeMillis(); - while(!lockRegistry.add(id)) + while (!lockRegistry.add(id)) { try { @@ -3178,9 +3198,9 @@ public class SolrInformationServer implements InformationServer // I don't think we are concerned with this exception. } - if(System.currentTimeMillis() - startTime > 120000) + if (System.currentTimeMillis() - startTime > 120000) { - throw new IOException("Unable to acquire lock on nodeId " + id + " after " + 120000 + " msecs."); + throw new AlfrescoLockException("Unable to acquire lock on nodeId " + id + " after " + 120000 + " msecs."); } } } @@ -3442,6 +3462,14 @@ public class SolrInformationServer implements InformationServer LOGGER.debug("No child doc found to update {}", childId); } } + catch (AlfrescoLockException exception) + { + LOGGER.error(exception.getMessage()); + } + catch (Exception exception) + { + LOGGER.error("Cascade update failure on child document {}. See the stacktrace below for further details.", childId, exception); + } finally { unlock(nodeId); From 55a251bb0d300fc22c661afdf013196e4b7bc8d3 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Tue, 14 Apr 2020 21:46:51 +0200 Subject: [PATCH 11/54] [ SEARCH-2175 ] Fix build errors --- .../main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java index c8f3976c3..a87a83b26 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java @@ -285,7 +285,8 @@ public class SOLRAPIQueueClient extends SOLRAPIClient return allNodes; } - public List getNodesMetaData(NodeMetaDataParameters params, int maxResults) throws AuthenticationException, IOException, JSONException + @Override + public List getNodesMetaData(NodeMetaDataParameters params) throws AuthenticationException, IOException, JSONException { if(throwException) { throw new ConnectException("THROWING EXCEPTION, better be ready!"); From 179de0ef2a8f5d8c5591af931638c8c3f58e7683 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Tue, 14 Apr 2020 23:47:15 +0200 Subject: [PATCH 12/54] [ SEARCH-2175 ] Fix build errors --- .../java/org/alfresco/solr/SolrInformationServer.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 95a596224..4f96ec119 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -1997,14 +1997,6 @@ public class SolrInformationServer implements InformationServer long end = System.nanoTime(); this.trackerStats.addNodeTime(end - start); } - catch (AlfrescoLockException exception) - { - LOGGER.error(exception.getMessage()); - } - catch (Exception exception) - { - LOGGER.error("Upsert failure on Node {}. See the stacktrace below for further details.", nodeId, exception); - } finally { unlock(nodeId); From 97a951f085c0520737cf402b5b67b2fe07e4ca4f Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 15 Apr 2020 06:03:40 +0200 Subject: [PATCH 13/54] [ SEARCH-2175 ] Don't catch exception on massive node deletetion --- .../java/org/alfresco/solr/SolrInformationServer.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 4f96ec119..7991889a6 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -1892,14 +1892,6 @@ public class SolrInformationServer implements InformationServer solrContentStore.removeDocFromContentStore(nodeMetaData); } - catch (AlfrescoLockException exception) - { - LOGGER.error(exception.getMessage()); - } - catch (Exception exception) - { - LOGGER.error("Unable to remove document {} from the content store. See the stacktrace below for further details.", nodeMetaData.getId(), exception); - } finally { unlock(nodeMetaData.getId()); From a2664c96d57a395700cb0d940a834e5cf6b9fa12 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 15 Apr 2020 06:33:17 +0200 Subject: [PATCH 14/54] [ SEARCH-2175 ] Pass MaxResults to Repo API as Int --- .../src/main/java/org/alfresco/solr/client/SOLRAPIClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 51864069b..34ed2623d 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -803,7 +803,7 @@ public class SOLRAPIClient if (params.getMaxResults().isPresent()) { - body.put("maxResults", params.getMaxResults()); + body.put("maxResults", params.getMaxResults().getAsInt()); } PostRequest req = new PostRequest(url.toString(), body.toString(), "application/json"); From 9e302ddda2a15fce23d38be77b2931d1af95e84d Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 15 Apr 2020 09:33:06 +0200 Subject: [PATCH 15/54] [ SEARCH-2187 ] First draft on wrapping callRepository with a Callable statement --- .../alfresco/solr/client/SOLRAPIClient.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 34ed2623d..d5cec7812 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -25,6 +25,7 @@ */ package org.alfresco.solr.client; +import static java.util.Collections.singletonList; import static java.util.Optional.ofNullable; import java.io.Closeable; @@ -42,6 +43,12 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.AlfrescoHttpClient; @@ -75,6 +82,7 @@ import org.alfresco.util.Pair; import org.apache.commons.codec.EncoderException; import org.apache.commons.codec.net.URLCodec; import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.httpclient.SimpleHttpConnectionManager; import org.apache.commons.httpclient.util.DateUtil; import org.json.JSONArray; import org.json.JSONException; @@ -1579,6 +1587,58 @@ public class SOLRAPIClient public void close() { repositoryHttpClient.close(); + executor.shutdown(); + } + + final ExecutorService executor = Executors.newSingleThreadExecutor(); + + private JSONObject callRepositoryWithTimeout(String msgId, Request req) throws IOException, AuthenticationException, InterruptedException, TimeoutException, ExecutionException { + List> result = executor.invokeAll(singletonList(() -> { + Response response = null; + LookAheadBufferedReader reader = null; + JSONObject json; + try + { + response = repositoryHttpClient.sendRequest(req); + if (response.getStatus() != HttpStatus.SC_OK) + { + throw new AlfrescoRuntimeException(msgId + " return status:" + response.getStatus()); + } + + reader = new LookAheadBufferedReader(new InputStreamReader(response.getContentAsStream(), StandardCharsets.UTF_8), LOGGER); + json = new JSONObject(new JSONTokener(reader)); + + if (LOGGER.isDebugEnabled()) + { + LOGGER.debug(json.toString(3)); + } + return json; + } + catch (JSONException exception) + { + String message = "Received a malformed JSON payload. Request was \"" + + req.getFullUri() + + "Data: " + + ofNullable(reader) + .map(LookAheadBufferedReader::lookAheadAndGetBufferedContent) + .orElse("Not available"); + LOGGER.error(message); + throw exception; + } + finally + { + ofNullable(response).ifPresent(Response::release); + ofNullable(reader).ifPresent(this::silentlyClose); + } + }), 5, TimeUnit.SECONDS); + + Future response = result.iterator().next(); + if(response.isCancelled()) + { + throw new TimeoutException("Request " + req + "has timed out. It has taken more than 5 seconds to respond"); + } + + return response.get(); } private JSONObject callRepository(String msgId, Request req) throws IOException, AuthenticationException From 0f0576812464d3f3e8bab19a541afcc769cb4006 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 15 Apr 2020 11:59:43 +0200 Subject: [PATCH 16/54] [ SEARCH-2182 ] Simplest approach for logging core, tracker type, instance and iteration id --- .../solr/tracker/AbstractTracker.java | 37 +++++++++++-------- .../org/alfresco/solr/tracker/AclTracker.java | 2 +- .../alfresco/solr/tracker/CascadeTracker.java | 24 +++++++++--- .../alfresco/solr/tracker/CommitTracker.java | 2 +- .../alfresco/solr/tracker/ContentTracker.java | 2 +- .../solr/tracker/CoreStatePublisher.java | 6 +-- .../solr/tracker/MetadataTracker.java | 2 +- .../alfresco/solr/tracker/ModelTracker.java | 10 ++--- .../solr/tracker/SlaveCoreStatePublisher.java | 4 +- .../solr/tracker/ContentTrackerIT.java | 4 +- .../solr/tracker/MetadataTrackerTest.java | 4 +- .../alfresco/solr/tracker/ModelTrackerIT.java | 2 +- 12 files changed, 58 insertions(+), 41 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java index 47930d6c0..d0a19474f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java @@ -20,7 +20,6 @@ package org.alfresco.solr.tracker; import static java.util.Optional.ofNullable; -import java.lang.invoke.MethodHandles; import java.net.ConnectException; import java.net.SocketTimeoutException; import java.util.Properties; @@ -45,7 +44,7 @@ public abstract class AbstractTracker implements Tracker static final long TIME_STEP_1_HR_IN_MS = 60 * 60 * 1000L; static final String SHARD_METHOD_DBID = "DB_ID"; - protected final static Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected final Logger logger = LoggerFactory.getLogger(getClass()); protected Properties props; protected SOLRAPIClient client; @@ -73,6 +72,7 @@ public abstract class AbstractTracker implements Tracker */ protected Throwable rollbackCausedBy; protected final Type type; + protected final String trackerId; /* * A thread handler can be used by subclasses, but they have to intentionally instantiate it. @@ -85,8 +85,9 @@ public abstract class AbstractTracker implements Tracker protected AbstractTracker(Type type) { this.type = type; + this.trackerId = type + "@" + hashCode(); } - + protected AbstractTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer,Type type) { this.props = p; @@ -109,6 +110,8 @@ public abstract class AbstractTracker implements Tracker this.trackerStats = this.infoSrv.getTrackerStats(); this.type = type; + + this.trackerId = type + "@" + hashCode(); } @@ -121,8 +124,10 @@ public abstract class AbstractTracker implements Tracker *
  • Index
    • *
  • Track repository
    • * + * + * @param iterationId an identifier which is uniquely associated with a given iteration. */ - protected abstract void doTrack() throws Throwable; + protected abstract void doTrack(String iterationId) throws Throwable; private boolean assertTrackerStateRemainsNull() { @@ -156,14 +161,16 @@ public abstract class AbstractTracker implements Tracker } /** * Template method - subclasses must implement the {@link Tracker}-specific indexing - * by implementing the abstract method {@link #doTrack()}. + * by implementing the abstract method {@link #doTrack(String)}. */ @Override public void track() { + String iterationId = "IT #" + System.currentTimeMillis(); + if(runLock.availablePermits() == 0) { - LOGGER.info("... {} for core [{}] is already in use {}", this.getClass().getSimpleName(), coreName, getClass()); + logger.info("[{} / {} / {}] Tracker already registered.", coreName, trackerId, iterationId); return; } @@ -180,13 +187,11 @@ public abstract class AbstractTracker implements Tracker assert(assertTrackerStateRemainsNull()); } - LOGGER.info("[CORE {}] Running {}", coreName, this.getClass().getSimpleName()); - if(this.state == null) { this.state = getTrackerState(); - LOGGER.debug("[CORE {}] Global Tracker State set to: {}", coreName, this.state.toString()); + logger.debug("[{} / {} / {}] Global Tracker State set to: {}", coreName, trackerId, iterationId, this.state.toString()); this.state.setRunning(true); } else @@ -199,33 +204,33 @@ public abstract class AbstractTracker implements Tracker try { - doTrack(); + doTrack(iterationId); } catch(IndexTrackingShutdownException t) { setRollback(true, t); - LOGGER.info("[CORE {}] Stopping index tracking for {}", coreName, getClass().getSimpleName()); + logger.info("[{} / {} / {}] Tracking cycle stopped. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); } catch(Throwable t) { setRollback(true, t); if (t instanceof SocketTimeoutException || t instanceof ConnectException) { - LOGGER.warn("[CORE {}] Tracking communication timed out for {}", coreName, getClass().getSimpleName()); - if (LOGGER.isDebugEnabled()) + logger.warn("[{} / {} / {}] Tracking communication timed out. See the stacktrace below for further details.", coreName, trackerId, iterationId); + if (logger.isDebugEnabled()) { - LOGGER.debug("[CORE {}] Stack trace", coreName, t); + logger.debug("[{} / {} / {}] Stack trace", coreName, trackerId, iterationId, t); } } else { - LOGGER.error("[CORE {}] Tracking failed for {}", coreName, getClass().getSimpleName(), t); + logger.error("[{} / {} / {}] Tracking failure. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); } } } catch (InterruptedException e) { - LOGGER.error("[CORE {}] Semaphore interrupted for {}", coreName, getClass().getSimpleName(), e); + logger.error("[{} / {} / {}] Semaphore interruption. See the stacktrace below for further details.", coreName, trackerId, iterationId, e); } finally { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java index e28785c7a..722e6ba26 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java @@ -90,7 +90,7 @@ public class AclTracker extends AbstractTracker } @Override - protected void doTrack() throws Throwable + protected void doTrack(String iterationId) throws Throwable { trackRepository(); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java index 18e79394f..6396500d4 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java @@ -36,6 +36,9 @@ import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.util.stream.Collectors.joining; +import static org.alfresco.solr.utils.Utils.notNullOrEmpty; + /* * This tracks Cascading Updates * @author Joel Bernstein @@ -61,13 +64,13 @@ public class CascadeTracker extends AbstractTracker implements Tracker } @Override - protected void doTrack() throws AuthenticationException, IOException, JSONException, EncoderException + protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException, EncoderException { // MetadataTracker must wait until ModelTracker has run ModelTracker modelTracker = this.infoSrv.getAdminHandler().getTrackerRegistry().getModelTracker(); if (modelTracker != null && modelTracker.hasModels()) { - trackRepository(); + trackRepository(iterationId); } } @@ -79,10 +82,10 @@ public class CascadeTracker extends AbstractTracker implements Tracker return false; } - private void trackRepository() throws IOException, AuthenticationException, JSONException, EncoderException + private void trackRepository(String iterationId) throws IOException, AuthenticationException, JSONException, EncoderException { checkShutdown(); - processCascades(); + processCascades(iterationId); } private void updateTransactionsAfterAsynchronous(List txsIndexed) @@ -125,9 +128,8 @@ public class CascadeTracker extends AbstractTracker implements Tracker infoSrv.setCleanCascadeTxnFloor(-1); } - private void processCascades() throws IOException + private void processCascades(String iterationId) throws IOException { - //System.out.println("######### processCascades()"); int num = 50; List txBatch = null; do { @@ -160,7 +162,17 @@ public class CascadeTracker extends AbstractTracker implements Tracker batch.add(stack.removeFirst()); } + CascadeIndexWorkerRunnable worker = new CascadeIndexWorkerRunnable(this.threadHandler, batch, infoSrv); + + if (logger.isTraceEnabled()) + { + String nodes = notNullOrEmpty(batch).stream() + .map(NodeMetaData::getId) + .map(Object::toString) + .collect(joining(",")); + logger.trace("[{} / {} / {} / {}] Worker has been created for nodes {}", coreName, trackerId, iterationId, worker.hashCode(), nodes); + } this.threadHandler.scheduleTask(worker); } while (stack.size() > 0); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java index e087752b2..fa2b2ee06 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java @@ -101,7 +101,7 @@ public class CommitTracker extends AbstractTracker } @Override - protected void doTrack() throws Throwable + protected void doTrack(String iterationId) throws Throwable { long currentTime = System.currentTimeMillis(); boolean commitNeeded = false; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java index e4b837280..534b1d7e2 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java @@ -57,7 +57,7 @@ public class ContentTracker extends AbstractTracker implements Tracker } @Override - protected void doTrack() throws Exception + protected void doTrack(String iterationId) throws Exception { //System.out.println("############## Content Tracker doTrack()"); try { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java index 591863964..aec77956c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java @@ -118,7 +118,7 @@ public abstract class CoreStatePublisher extends AbstractTracker updateShardProperty(); if (shardProperty.isEmpty()) { - LOGGER.warn("Sharding property {} was set to {}, but no such property was found.", SHARD_KEY_KEY, shardKeyName); + logger.warn("Sharding property {} was set to {}, but no such property was found.", SHARD_KEY_KEY, shardKeyName); } }); } @@ -134,11 +134,11 @@ public abstract class CoreStatePublisher extends AbstractTracker { if (updatedShardProperty.isEmpty()) { - LOGGER.warn("The model defining {} property has been disabled", shardKeyName); + logger.warn("The model defining {} property has been disabled", shardKeyName); } else { - LOGGER.info("New {} property found for {} ", SHARD_KEY_KEY, shardKeyName); + logger.info("New {} property found for {} ", SHARD_KEY_KEY, shardKeyName); } } shardProperty = updatedShardProperty; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 8636c9c71..551599fef 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -169,7 +169,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } @Override - protected void doTrack() throws AuthenticationException, IOException, JSONException, EncoderException + protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException, EncoderException { log.debug("### MetadataTracker doTrack ###"); // MetadataTracker must wait until ModelTracker has run diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java index 034d14228..946634c3c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java @@ -106,7 +106,7 @@ public class ModelTracker extends AbstractTracker implements Tracker super(p, client, coreName, informationServer, Tracker.Type.MODEL); String normalSolrHome = SolrResourceLoader.normalizeDir(solrHome); alfrescoModelDir = new File(ConfigUtil.locateProperty("solr.model.dir", normalSolrHome+"alfrescoModels")); - LOGGER.info("Alfresco Model dir " + alfrescoModelDir); + logger.info("Alfresco Model dir " + alfrescoModelDir); if (!alfrescoModelDir.exists()) { alfrescoModelDir.mkdir(); @@ -191,13 +191,13 @@ public class ModelTracker extends AbstractTracker implements Tracker } @Override - protected void doTrack() throws AuthenticationException, IOException, JSONException + protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException { // Is the InformationServer ready to update int registeredSearcherCount = this.infoSrv.getRegisteredSearcherCount(); if (registeredSearcherCount >= getMaxLiveSearchers()) { - LOGGER.info(".... skipping tracking registered searcher count = " + registeredSearcherCount); + logger.info(".... skipping tracking registered searcher count = " + registeredSearcherCount); return; } @@ -268,7 +268,7 @@ public class ModelTracker extends AbstractTracker implements Tracker } catch (Throwable t) { - LOGGER.error("Model tracking failed for core: "+ coreName, t); + logger.error("Model tracking failed for core: "+ coreName, t); } } @@ -534,7 +534,7 @@ public class ModelTracker extends AbstractTracker implements Tracker { loadedModels.add(modelName); } - LOGGER.info("Loading model " + model.getName()); + logger.info("Loading model " + model.getName()); } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java index 71750c53f..39e4ff3d2 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java @@ -58,7 +58,7 @@ public class SlaveCoreStatePublisher extends CoreStatePublisher } @Override - protected void doTrack() + protected void doTrack(String iterationId) { try { @@ -67,7 +67,7 @@ public class SlaveCoreStatePublisher extends CoreStatePublisher } catch (EncoderException | IOException | AuthenticationException exception ) { - LOGGER.error("Unable to publish this node state. " + + logger.error("Unable to publish this node state. " + "A failure condition has been met during the outbound subscription message encoding process. " + "See the stacktrace below for further details.", exception); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java index 517a5f241..b3e713665 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java @@ -71,7 +71,7 @@ public class ContentTrackerIT public void doTrackWithNoContentDoesNothing() throws Exception { - this.contentTracker.doTrack(); + this.contentTracker.doTrack("anIterationId"); verify(srv, never()).updateContentToIndexAndCache(anyLong(), anyString()); verify(srv, never()).commit(); } @@ -107,7 +107,7 @@ public class ContentTrackerIT .thenReturn(docs1) .thenReturn(docs2) .thenReturn(emptyList); - this.contentTracker.doTrack(); + this.contentTracker.doTrack("anIterationId"); InOrder order = inOrder(srv); order.verify(srv).getDocsWithUncleanContent(0, READ_BATCH); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java index 7d2c4c1ce..b3b4d56fd 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java @@ -117,7 +117,7 @@ public class MetadataTrackerTest nodes.add(node ); when(repositoryClient.getNodes(any(GetNodesParameters.class), anyInt())).thenReturn(nodes); - this.metadataTracker.doTrack(); + this.metadataTracker.doTrack("AnIterationId"); InOrder inOrder = inOrder(srv); inOrder.verify(srv).indexNodes(nodes, true); @@ -141,7 +141,7 @@ public class MetadataTrackerTest when(repositoryClient.getTransactions(anyLong(), anyLong(), anyLong(), anyLong(), anyInt(), any(ShardState.class))).thenReturn(txs); when(repositoryClient.getTransactions(anyLong(), anyLong(), anyLong(), anyLong(), anyInt())).thenReturn(txs); - this.metadataTracker.doTrack(); + this.metadataTracker.doTrack("AnIterationId"); verify(srv, never()).commit(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ModelTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ModelTrackerIT.java index b0a64fe97..5259f300b 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ModelTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ModelTrackerIT.java @@ -118,7 +118,7 @@ public class ModelTrackerIT public void testDoTrack() throws AuthenticationException, IOException, JSONException { ModelTracker spiedModelTracker = spy(this.modelTracker); - spiedModelTracker.doTrack(); + spiedModelTracker.doTrack("AnIterationId"); verify(this.srv).getRegisteredSearcherCount(); verify(spiedModelTracker).trackModels(false); From 78af6df57c73bb74c542b6331436ef655ca24d30 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Wed, 15 Apr 2020 12:41:57 +0200 Subject: [PATCH 17/54] SEARCH-950: Test field identifiers containing hyphens (-) for SQL Select queries --- e2e-test/src/main/resources/model/music-model.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-test/src/main/resources/model/music-model.xml b/e2e-test/src/main/resources/model/music-model.xml index 9406e7e78..23c01e337 100644 --- a/e2e-test/src/main/resources/model/music-model.xml +++ b/e2e-test/src/main/resources/model/music-model.xml @@ -29,7 +29,7 @@ d:text false - TRUE + BOTH false @@ -66,7 +66,7 @@ d:text false - TRUE + BOTH false From cb50bab3572135b51eb9e5e93176b674ce7f86fe Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 15 Apr 2020 14:26:48 +0200 Subject: [PATCH 18/54] [ SEARCH-2187 ] acquire lock max wait time, configuration property --- .../alfresco/solr/SolrInformationServer.java | 8 ++- .../templates/rerank/conf/solrcore.properties | 2 + .../alfresco/solr/client/SOLRAPIClient.java | 52 ------------------- 3 files changed, 8 insertions(+), 54 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 7991889a6..f8f617118 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -296,6 +296,8 @@ public class SolrInformationServer implements InformationServer private String skippingDocsQueryString; private boolean isSkippingDocsInitialized; + private long maxAllowedTimeForAcquiringDbIdLock; + protected enum FTSStatus {New, Dirty, Clean} static class DocListCollector implements Collector, LeafCollector @@ -461,6 +463,8 @@ public class SolrInformationServer implements InformationServer holeRetention = Integer.parseInt(p.getProperty("alfresco.hole.retention", "3600000")); minHash = Boolean.parseBoolean(p.getProperty("alfresco.fingerprint", "true")); + maxAllowedTimeForAcquiringDbIdLock = Long.parseLong(p.getProperty("alfresco.tracker.maxNodeLockMs", "120000")); + dataModel = AlfrescoSolrDataModel.getInstance(); contentStreamLimit = Integer.parseInt(p.getProperty("alfresco.contentStreamLimit", "10000000")); @@ -3182,9 +3186,9 @@ public class SolrInformationServer implements InformationServer // I don't think we are concerned with this exception. } - if (System.currentTimeMillis() - startTime > 120000) + if (System.currentTimeMillis() - startTime > maxAllowedTimeForAcquiringDbIdLock) { - throw new AlfrescoLockException("Unable to acquire lock on nodeId " + id + " after " + 120000 + " msecs."); + throw new AlfrescoLockException("Unable to acquire lock on nodeId " + id + " after " + maxAllowedTimeForAcquiringDbIdLock + " msecs."); } } } diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index 1d33062d9..b9771edb8 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -30,6 +30,8 @@ alfresco.hole.check.after=300000 alfresco.batch.count=5000 alfresco.recordUnindexedNodes=false +# max time (in msecs) a given tracker instance will try to acquire a lock on a given DBID +alfresco.tracker.maxNodeLockMs=120000 # encryption # none, https diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index d5cec7812..760eba485 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -1587,58 +1587,6 @@ public class SOLRAPIClient public void close() { repositoryHttpClient.close(); - executor.shutdown(); - } - - final ExecutorService executor = Executors.newSingleThreadExecutor(); - - private JSONObject callRepositoryWithTimeout(String msgId, Request req) throws IOException, AuthenticationException, InterruptedException, TimeoutException, ExecutionException { - List> result = executor.invokeAll(singletonList(() -> { - Response response = null; - LookAheadBufferedReader reader = null; - JSONObject json; - try - { - response = repositoryHttpClient.sendRequest(req); - if (response.getStatus() != HttpStatus.SC_OK) - { - throw new AlfrescoRuntimeException(msgId + " return status:" + response.getStatus()); - } - - reader = new LookAheadBufferedReader(new InputStreamReader(response.getContentAsStream(), StandardCharsets.UTF_8), LOGGER); - json = new JSONObject(new JSONTokener(reader)); - - if (LOGGER.isDebugEnabled()) - { - LOGGER.debug(json.toString(3)); - } - return json; - } - catch (JSONException exception) - { - String message = "Received a malformed JSON payload. Request was \"" + - req.getFullUri() + - "Data: " - + ofNullable(reader) - .map(LookAheadBufferedReader::lookAheadAndGetBufferedContent) - .orElse("Not available"); - LOGGER.error(message); - throw exception; - } - finally - { - ofNullable(response).ifPresent(Response::release); - ofNullable(reader).ifPresent(this::silentlyClose); - } - }), 5, TimeUnit.SECONDS); - - Future response = result.iterator().next(); - if(response.isCancelled()) - { - throw new TimeoutException("Request " + req + "has timed out. It has taken more than 5 seconds to respond"); - } - - return response.get(); } private JSONObject callRepository(String msgId, Request req) throws IOException, AuthenticationException From abc0cb2195b8cd51bab497bdec454adcc4175d32 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 15 Apr 2020 14:29:34 +0200 Subject: [PATCH 19/54] [ SEARCH-2187 ] removed unused import on SOLRAPIClient --- .../alfresco/solr/client/SOLRAPIClient.java | 49 ++++++++----------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 760eba485..b141929d4 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -25,31 +25,9 @@ */ package org.alfresco.solr.client; -import static java.util.Collections.singletonList; -import static java.util.Optional.ofNullable; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.AlfrescoHttpClient; import org.alfresco.httpclient.AuthenticationException; @@ -82,7 +60,6 @@ import org.alfresco.util.Pair; import org.apache.commons.codec.EncoderException; import org.apache.commons.codec.net.URLCodec; import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.httpclient.SimpleHttpConnectionManager; import org.apache.commons.httpclient.util.DateUtil; import org.json.JSONArray; import org.json.JSONException; @@ -92,9 +69,23 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.extensions.surf.util.URLEncoder; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import static java.util.Optional.ofNullable; // TODO error handling, including dealing with a repository that is not responsive (ConnectException in sendRemoteRequest) // TODO get text content transform status handling From 527586fa3fdac18b8e9e611e66a3311e9eafb33c Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 15 Apr 2020 14:41:31 +0200 Subject: [PATCH 20/54] [ SEARCH-2187 ] same configuration property on noRerank template --- .../solr/instance/templates/noRerank/conf/solrcore.properties | 3 +++ .../solr/instance/templates/rerank/conf/solrcore.properties | 1 + 2 files changed, 4 insertions(+) diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties index 189c181e0..33b09117e 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties @@ -30,6 +30,9 @@ alfresco.hole.check.after=300000 alfresco.batch.count=5000 alfresco.recordUnindexedNodes=false +# max time (in msecs) a given tracker instance will try to acquire a lock on a given DBID +alfresco.tracker.maxNodeLockMs=120000 + # encryption # none, https diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index b9771edb8..6209e74cf 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -32,6 +32,7 @@ alfresco.recordUnindexedNodes=false # max time (in msecs) a given tracker instance will try to acquire a lock on a given DBID alfresco.tracker.maxNodeLockMs=120000 + # encryption # none, https From 8d4f5e357daede69a62be8737ceed8c429d35298 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Mon, 20 Apr 2020 09:18:28 +0200 Subject: [PATCH 21/54] SEARCH-950: Moved utility functions to Abstract Class. Model modified to test both TRUE and BOTH tokenization values. --- e2e-test/src/main/resources/model/music-model.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/src/main/resources/model/music-model.xml b/e2e-test/src/main/resources/model/music-model.xml index 23c01e337..d7de00d47 100644 --- a/e2e-test/src/main/resources/model/music-model.xml +++ b/e2e-test/src/main/resources/model/music-model.xml @@ -29,7 +29,7 @@ d:text false - BOTH + TRUE false From 8f72f71f16b96f9805c376d0effde5fe6bc95fe3 Mon Sep 17 00:00:00 2001 From: mbhave Date: Tue, 21 Apr 2020 10:40:43 +0100 Subject: [PATCH 22/54] Search-2121: Add Docker setting to disable Cascade tracking --- search-services/packaging/src/docker/6.x/docker-compose.yml | 2 ++ search-services/packaging/src/docker/search_config_setup.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/search-services/packaging/src/docker/6.x/docker-compose.yml b/search-services/packaging/src/docker/6.x/docker-compose.yml index 13b5861e9..ffa3f4133 100644 --- a/search-services/packaging/src/docker/6.x/docker-compose.yml +++ b/search-services/packaging/src/docker/6.x/docker-compose.yml @@ -70,6 +70,8 @@ services: - ALFRESCO_SECURE_COMMS=none #Enable Spellcheck by setting to true - ENABLE_SPELLCHECK=${SEARCH_ENABLE_SPELLCHECK} + #Disable cascade tracking by setting to true + - DISABLE_CASCADE_TRACKING=${DISABLE_CASCADE_TRACKING} ports: - 8083:8983 #Browser port diff --git a/search-services/packaging/src/docker/search_config_setup.sh b/search-services/packaging/src/docker/search_config_setup.sh index 0d2ef18a1..35c1360d8 100644 --- a/search-services/packaging/src/docker/search_config_setup.sh +++ b/search-services/packaging/src/docker/search_config_setup.sh @@ -101,4 +101,8 @@ if [[ true == "$ENABLE_SPELLCHECK" ]]; then sed -i 's/#alfresco.suggestable.property/alfresco.suggestable.property/' ${PWD}/solrhome/conf/shared.properties fi +if [[ true == "$DISABLE_CASCADE_TRACKING" ]]; then + sed -i 's/alfresco.cascade.tracker.enabled=true/alfresco.cascade.tracker.enabled=false/' ${PWD}/solrhome/conf/shared.properties +fi + bash -c "$@" \ No newline at end of file From e7e08e3757253c912831e8694cbf6168a162d4ad Mon Sep 17 00:00:00 2001 From: mbhave Date: Tue, 21 Apr 2020 10:43:42 +0100 Subject: [PATCH 23/54] Updated comment to suggest default for disable cascade tracking --- search-services/packaging/src/docker/6.x/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/packaging/src/docker/6.x/docker-compose.yml b/search-services/packaging/src/docker/6.x/docker-compose.yml index ffa3f4133..afb2633e4 100644 --- a/search-services/packaging/src/docker/6.x/docker-compose.yml +++ b/search-services/packaging/src/docker/6.x/docker-compose.yml @@ -70,7 +70,7 @@ services: - ALFRESCO_SECURE_COMMS=none #Enable Spellcheck by setting to true - ENABLE_SPELLCHECK=${SEARCH_ENABLE_SPELLCHECK} - #Disable cascade tracking by setting to true + #Disable cascade tracking by setting to true, default false - as cascade tracking is enabled. - DISABLE_CASCADE_TRACKING=${DISABLE_CASCADE_TRACKING} ports: - 8083:8983 #Browser port From ee8a899893859177ba7296efb7d3a58c0e2e4c0f Mon Sep 17 00:00:00 2001 From: mbhave Date: Tue, 21 Apr 2020 11:42:49 +0100 Subject: [PATCH 24/54] Search-2121: Added new test group to tag the tests to be excluded when cascade tracker is disabled --- e2e-test/src/main/java/org/alfresco/search/TestGroup.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e-test/src/main/java/org/alfresco/search/TestGroup.java b/e2e-test/src/main/java/org/alfresco/search/TestGroup.java index 8da8c14e8..f08879c17 100644 --- a/e2e-test/src/main/java/org/alfresco/search/TestGroup.java +++ b/e2e-test/src/main/java/org/alfresco/search/TestGroup.java @@ -34,6 +34,8 @@ public class TestGroup public static final String CONFIG_SHARDING_EXPLICIT ="CONFIG_Sharding_EXPLICIT"; // Alfresco search services using sharded environment and explicit routing public static final String CONFIG_SHARDING_DB_ID_RANGE = "CONFIG_Sharding_DB_ID_RANGE"; // Alfresco Search Services using Sharding with DB_ID_RANGE + public static final String CONFIG_DISABLE_CASCADE_TRACKER ="Config_Disable_Cascade_Tracker"; // Alfresco search services does not index fields related to cascaded updates + public static final String NOT_INSIGHT_ENGINE = "Not_InsightEngine"; // When Alfresco Insight Engine 1.0 isn't running public static final String ACS_52n = "ACS_52n"; // Alfresco Content Services 5.2.n From 3b3a6b26770d982d2f4d7986d5b96ee19cb54250 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2020 21:12:00 +0000 Subject: [PATCH 25/54] Bump alfresco-data-model from 8.102 to 8.105 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.102 to 8.105. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.102...8.105) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 17476b908..d03a5a378 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.102 + 8.105 2.10.3 From 971becb9055eedc5e1162b28ec9963154d74446e Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Apr 2020 10:15:03 +0100 Subject: [PATCH 26/54] SEARCH-2202 Avoid returning mutable list. --- .../main/java/org/alfresco/solr/AlfrescoSolrDataModel.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java index 50b6d9563..2fbdf8976 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java @@ -18,6 +18,8 @@ */ package org.alfresco.solr; +import static java.util.Collections.unmodifiableList; + import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -1339,7 +1341,7 @@ public class AlfrescoSolrDataModel implements QueryConstants public List getFields() { - return fields; + return unmodifiableList(fields); } public void addField(String prefix, boolean localised, boolean sort) From cbce32415e2cfa33eec3ab70622d466427093c6a Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Apr 2020 10:19:09 +0100 Subject: [PATCH 27/54] SEARCH-2202 Avoid storing/returning mutable deque. --- .../java/org/alfresco/solr/BoundedDeque.java | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/BoundedDeque.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/BoundedDeque.java index 8c83553f6..9717dc89a 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/BoundedDeque.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/BoundedDeque.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Solr Client - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Solr Client + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.solr; @@ -38,7 +38,7 @@ public class BoundedDeque implements Iterable public BoundedDeque(int max) { this.max = max; - setDeque(new LinkedBlockingDeque()); + this.deque = new LinkedBlockingDeque(); } /** @@ -46,21 +46,21 @@ public class BoundedDeque implements Iterable */ public int size() { - return getDeque().size(); + return deque.size(); } public void add(T add) { - while (getDeque().size() > (max - 1)) + while (deque.size() > (max - 1)) { - getDeque().removeLast(); + deque.removeLast(); } - getDeque().addFirst(add); + deque.addFirst(add); } public T getLast() { - return getDeque().getFirst(); + return deque.getFirst(); } /* @@ -70,17 +70,17 @@ public class BoundedDeque implements Iterable @Override public Iterator iterator() { - return getDeque().iterator(); + return deque.iterator(); } + /** Get a copy of the deque. */ public LinkedBlockingDeque getDeque() { - return deque; + return new LinkedBlockingDeque(deque); } public void setDeque(LinkedBlockingDeque deque) { - this.deque = deque; + this.deque = new LinkedBlockingDeque(deque); } - -} \ No newline at end of file +} From 6c50bd0f5e0161bbe7c06e0bea5a27379f2d6dfe Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Apr 2020 10:22:34 +0100 Subject: [PATCH 28/54] SEARCH-2202 Making constant protected. --- .../src/main/java/org/alfresco/solr/HandlerOfResources.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerOfResources.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerOfResources.java index aedad61fe..9d8052652 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerOfResources.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerOfResources.java @@ -40,7 +40,7 @@ import org.apache.solr.common.params.SolrParams; */ public class HandlerOfResources { - public static final List DISALLOWED_SHARED_UPDATES = Arrays.asList("alfresco.identifier.property.", + protected static final List DISALLOWED_SHARED_UPDATES = Arrays.asList("alfresco.identifier.property.", "alfresco.suggestable.property.", "alfresco.cross.locale.property.", "alfresco.cross.locale.datatype."); From 7b15c3ede737a5a87fd11e55d9e634ed4620c3e8 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Apr 2020 10:37:53 +0100 Subject: [PATCH 29/54] SEARCH-2202 Declare constants final. --- .../java/org/alfresco/solr/cache/CacheConstants.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/cache/CacheConstants.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/cache/CacheConstants.java index 7d578e99c..a16bde482 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/cache/CacheConstants.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/cache/CacheConstants.java @@ -32,9 +32,9 @@ public class CacheConstants // public static String ALFRESCO_READER_TO_ACL_IDS_CACHE = "alfrescoReaderToAclIdsCache"; // public static String ALFRESCO_DENY_TO_ACL_IDS_CACHE = "alfrescoDenyToAclIdsCache"; - public static String ALFRESCO_AUTHORITY_CACHE = "alfrescoAuthorityCache"; - public static String ALFRESCO_OWNERLOOKUP_CACHE = "alfrescoOwnerCache"; - public static String ALFRESCO_READER_CACHE = "alfrescoReaderCache"; - public static String ALFRESCO_DENIED_CACHE = "alfrescoDeniedCache"; - public static String ALFRESCO_PATH_CACHE = "alfrescoPathCache"; + public final static String ALFRESCO_AUTHORITY_CACHE = "alfrescoAuthorityCache"; + public final static String ALFRESCO_OWNERLOOKUP_CACHE = "alfrescoOwnerCache"; + public final static String ALFRESCO_READER_CACHE = "alfrescoReaderCache"; + public final static String ALFRESCO_DENIED_CACHE = "alfrescoDeniedCache"; + public final static String ALFRESCO_PATH_CACHE = "alfrescoPathCache"; } From dc019bcc0434860012049b83da8a6504aa69027d Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Apr 2020 17:06:44 +0100 Subject: [PATCH 30/54] SEARCH-2202 Make more list fields unmodifiable. Also fix integration tests to honour the encapsulation. --- .../AlfrescoSolrTrackerExceptionIT.java | 15 ++-- .../solr/tracker/AlfrescoSolrTrackerIT.java | 15 ++-- .../alfresco/solr/client/AclChangeSets.java | 56 +++++++-------- .../org/alfresco/solr/client/AclReaders.java | 69 +++++++++++-------- 4 files changed, 84 insertions(+), 71 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java index 43d779f8f..00f697a91 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java @@ -18,6 +18,8 @@ */ package org.alfresco.solr.tracker; +import static java.util.Arrays.asList; + import static org.alfresco.solr.AlfrescoSolrUtils.ancestors; import static org.alfresco.solr.AlfrescoSolrUtils.createGUID; import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; @@ -350,13 +352,11 @@ public class AlfrescoSolrTrackerExceptionIT extends AbstractAlfrescoSolrIT logger.info("#################### Passed Fifteenth Test ##############################"); - - List readers = aclReaders.getReaders(); - readers.set(0, "andy"); // Change the aclReader + // Change the aclReaders + aclReaders.setReaders(asList("andy")); indexAclId(acl.getId()); - List readers2 = aclReaders2.getReaders(); - readers2.set(0, "ice"); // Change the aclReader + aclReaders2.setReaders(asList("ice")); reindexAclId(acl2.getId()); @@ -379,8 +379,9 @@ public class AlfrescoSolrTrackerExceptionIT extends AbstractAlfrescoSolrIT logger.info("#################### Passed Seventeenth Test ##############################"); - readers.set(0, "alan"); // Change the aclReader - readers2.set(0, "paul"); // Change the aclReader + // Change the aclReaders + aclReaders.setReaders(asList("alan")); + aclReaders2.setReaders(asList("paul")); reindexAclChangeSetId(aclChangeSet.getId()); //This should replace "andy" and "ice" with "alan" and "paul" diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java index 749e82c03..b02400952 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java @@ -18,6 +18,8 @@ */ package org.alfresco.solr.tracker; +import static java.util.Arrays.asList; + import static org.alfresco.solr.AlfrescoSolrUtils.ancestors; import static org.alfresco.solr.AlfrescoSolrUtils.createGUID; import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; @@ -355,13 +357,11 @@ public class AlfrescoSolrTrackerIT extends AbstractAlfrescoSolrIT logger.info("#################### Passed Fifteenth Test ##############################"); - - List readers = aclReaders.getReaders(); - readers.set(0, "andy"); // Change the aclReader + // Change the aclReaders + aclReaders.setReaders(asList("andy")); indexAclId(acl.getId()); - List readers2 = aclReaders2.getReaders(); - readers2.set(0, "ice"); // Change the aclReader + aclReaders2.setReaders(asList("ice")); reindexAclId(acl2.getId()); @@ -384,8 +384,9 @@ public class AlfrescoSolrTrackerIT extends AbstractAlfrescoSolrIT logger.info("#################### Passed Seventeenth Test ##############################"); - readers.set(0, "alan"); // Change the aclReader - readers2.set(0, "paul"); // Change the aclReader + // Change the aclReaders + aclReaders.setReaders(asList("alan")); + aclReaders2.setReaders(asList("paul")); reindexAclChangeSetId(aclChangeSet.getId()); //This should replace "andy" and "ice" with "alan" and "paul" diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java index f5e9243b1..5a508b361 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java @@ -1,30 +1,32 @@ -/* - * #%L - * Alfresco Solr Client - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Solr Client + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.solr.client; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -41,14 +43,14 @@ public class AclChangeSets AclChangeSets(List aclChangeSets, Long maxChangeSetCommitTime, Long maxChangeSetId) { - this.aclChangeSets = aclChangeSets; + this.aclChangeSets = new ArrayList<>(aclChangeSets); this.maxChangeSetCommitTime = maxChangeSetCommitTime; this.maxChangeSetId = maxChangeSetId; } public List getAclChangeSets() { - return aclChangeSets; + return Collections.unmodifiableList(aclChangeSets); } public Long getMaxChangeSetCommitTime() diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclReaders.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclReaders.java index 015c5b2ad..ade01f21f 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclReaders.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclReaders.java @@ -1,30 +1,34 @@ -/* - * #%L - * Alfresco Solr Client - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Solr Client + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.solr.client; +import static java.util.Collections.unmodifiableList; + +import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -36,7 +40,7 @@ public class AclReaders { private final long id; - private final List readers; + private List readers; private final List denied; @@ -47,8 +51,8 @@ public class AclReaders public AclReaders(long id, List readers, List denied, long aclChangeSetId, String tenantDomain) { this.id = id; - this.readers = readers; - this.denied = denied; + this.readers = new ArrayList<>(readers); + this.denied = new ArrayList<>(denied); this.aclChangeSetId = aclChangeSetId; this.tenantDomain = tenantDomain; } @@ -96,12 +100,17 @@ public class AclReaders public List getReaders() { - return readers; + return unmodifiableList(readers); + } + + public void setReaders(List readers) + { + this.readers = new ArrayList<>(readers); } public List getDenied() { - return denied; + return unmodifiableList(denied); } public long getAclChangeSetId() From e0cee5c5e7078c0f5ca2eb2c6d6831307d8cbb32 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Apr 2020 15:56:58 +0100 Subject: [PATCH 31/54] SEARCH-2202 Fix NPE issues and add unit test for BoundedDeque. --- .../alfresco/solr/client/AclChangeSets.java | 2 +- .../org/alfresco/solr/client/AclReaders.java | 4 +- .../org/alfresco/solr/BoundedDequeTest.java | 57 +++++++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/BoundedDequeTest.java diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java index 5a508b361..1f1b2189f 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java @@ -43,7 +43,7 @@ public class AclChangeSets AclChangeSets(List aclChangeSets, Long maxChangeSetCommitTime, Long maxChangeSetId) { - this.aclChangeSets = new ArrayList<>(aclChangeSets); + this.aclChangeSets = (aclChangeSets == null ? null : new ArrayList<>(aclChangeSets)); this.maxChangeSetCommitTime = maxChangeSetCommitTime; this.maxChangeSetId = maxChangeSetId; } diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclReaders.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclReaders.java index ade01f21f..f18fc438f 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclReaders.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclReaders.java @@ -51,8 +51,8 @@ public class AclReaders public AclReaders(long id, List readers, List denied, long aclChangeSetId, String tenantDomain) { this.id = id; - this.readers = new ArrayList<>(readers); - this.denied = new ArrayList<>(denied); + this.readers = (readers == null ? null : new ArrayList<>(readers)); + this.denied = (denied == null ? null : new ArrayList<>(denied)); this.aclChangeSetId = aclChangeSetId; this.tenantDomain = tenantDomain; } diff --git a/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/BoundedDequeTest.java b/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/BoundedDequeTest.java new file mode 100644 index 000000000..73472f71a --- /dev/null +++ b/search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/BoundedDequeTest.java @@ -0,0 +1,57 @@ +/* + * #%L + * Alfresco Solr Client + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.solr; + +import static java.util.Arrays.asList; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.concurrent.LinkedBlockingDeque; + +import org.junit.Test; + +/** Unit tests for the {@link BoundedDeque}. */ +public class BoundedDequeTest +{ + /** Check that earlier entries are removed from the BoundedDeque. */ + @Test + public void testBoundedness() + { + // Create a BoundedDeque with size two. + BoundedDeque boundedDeque = new BoundedDeque<>(2); + + // Add three things. + boundedDeque.add("A"); + boundedDeque.add("B"); + boundedDeque.add("C"); + + // Check that the latest two are still there. + LinkedBlockingDeque actual = boundedDeque.getDeque(); + assertEquals("Unexpected entries in BoundedDeque.", asList("C", "B"), new ArrayList(actual)); + } +} From bcb97ddd0dc8a8266cbc6969164661b6ef0f57aa Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Apr 2020 16:25:42 +0100 Subject: [PATCH 32/54] SEARCH-2202 Use constants for fields in SOLRAPIQueueClient. These collections should be constants anyway - they're never set. --- .../org/alfresco/solr/AlfrescoSolrReloadIT.java | 6 +++--- .../org/alfresco/solr/AlfrescoSolrUtils.java | 7 +++---- .../solr/AlfrescoTrackerRegistrationIT.java | 6 +++--- .../org/alfresco/solr/SolrITInitializer.java | 8 +++----- .../solr/handler/ContentStoreReplicationIT.java | 6 +++--- .../solr/highlight/AlfrescoHighlighterIT.java | 6 +++--- .../solr/query/AlfrescoSolrFingerprintIT.java | 6 +++--- .../tracker/AlfrescoSolrTrackerExceptionIT.java | 6 +++--- .../solr/tracker/AlfrescoSolrTrackerIT.java | 6 +++--- .../tracker/AlfrescoSolrTrackerRollbackIT.java | 6 +++--- .../solr/tracker/AlfrescoSolrTrackerStateIT.java | 6 +++--- .../org/alfresco/solr/tracker/CascadingIT.java | 6 +++--- .../alfresco/solr/client/SOLRAPIQueueClient.java | 16 ++++++++-------- 13 files changed, 44 insertions(+), 47 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java index 70500adf6..a67431823 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java @@ -51,9 +51,9 @@ public class AlfrescoSolrReloadIT extends AbstractAlfrescoSolrIT public void clearQueue() throws Exception { SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java index 92f73d41b..aa7cb5ba6 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java @@ -89,7 +89,6 @@ import org.alfresco.solr.client.SOLRAPIQueueClient; import org.alfresco.solr.client.StringPropertyValue; import org.alfresco.solr.client.Transaction; import org.alfresco.util.ISO9075; -import org.alfresco.util.Pair; import org.apache.solr.SolrTestCaseJ4.XmlDoc; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CoreAdminParams; @@ -383,17 +382,17 @@ public class AlfrescoSolrUtils public static void indexAclChangeSet(AclChangeSet aclChangeSet, List aclList, List aclReadersList) { //First map the nodes to a transaction. - SOLRAPIQueueClient.aclMap.put(aclChangeSet.getId(), aclList); + SOLRAPIQueueClient.ACL_MAP.put(aclChangeSet.getId(), aclList); //Next map a node to the NodeMetaData for(AclReaders aclReaders : aclReadersList) { - SOLRAPIQueueClient.aclReadersMap.put(aclReaders.getId(), aclReaders); + SOLRAPIQueueClient.ACL_READERS_MAP.put(aclReaders.getId(), aclReaders); } //Next add the transaction to the queue - SOLRAPIQueueClient.aclChangeSetQueue.add(aclChangeSet); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.add(aclChangeSet); } /** * Generate a collection from input. diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationIT.java index b679ec5fe..b744df26a 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationIT.java @@ -41,9 +41,9 @@ public class AlfrescoTrackerRegistrationIT extends AbstractAlfrescoSolrIT public void clearQueue() throws Exception { SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); } @Test diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index 3bc5bd666..d73b3b2a9 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -4,7 +4,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; import org.alfresco.solr.basics.RandomSupplier; import org.alfresco.solr.client.SOLRAPIQueueClient; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.util.Time; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.embedded.JettyConfig; @@ -15,7 +14,6 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.eclipse.jetty.servlet.ServletHolder; import org.junit.BeforeClass; -import org.junit.rules.ExternalResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -224,9 +222,9 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java index 678afd5ad..14f3f1489 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java @@ -146,9 +146,9 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); SOLRAPIQueueClient.nodeContentMap.clear(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterIT.java index c19a8ee68..91f8d7e4f 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterIT.java @@ -154,9 +154,9 @@ public class AlfrescoHighlighterIT extends AbstractAlfrescoSolrIT public static void clearQueue() { SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); SOLRAPIQueueClient.nodeContentMap.clear(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrFingerprintIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrFingerprintIT.java index d9f0694e2..be091661f 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrFingerprintIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrFingerprintIT.java @@ -90,9 +90,9 @@ public class AlfrescoSolrFingerprintIT extends AbstractAlfrescoSolrIT { SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); SOLRAPIQueueClient.nodeContentMap.clear(); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java index 00f697a91..b2af29019 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java @@ -87,9 +87,9 @@ public class AlfrescoSolrTrackerExceptionIT extends AbstractAlfrescoSolrIT public void clearQueue() throws Exception { SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java index b02400952..3ac837bf4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java @@ -88,9 +88,9 @@ public class AlfrescoSolrTrackerIT extends AbstractAlfrescoSolrIT public void clearQueue() throws Exception { SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerRollbackIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerRollbackIT.java index 9baf19990..73da64f64 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerRollbackIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerRollbackIT.java @@ -80,9 +80,9 @@ public class AlfrescoSolrTrackerRollbackIT extends AbstractAlfrescoSolrIT public void clearQueue() throws Exception { SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java index 119aab737..078f6fa7e 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java @@ -82,9 +82,9 @@ public class AlfrescoSolrTrackerStateIT extends AbstractAlfrescoSolrIT public void clearQueue() throws Exception { SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadingIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadingIT.java index 7d1e145a9..b14d792ad 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadingIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadingIT.java @@ -152,9 +152,9 @@ public class CascadingIT extends AbstractAlfrescoSolrIT { SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); - SOLRAPIQueueClient.aclChangeSetQueue.clear(); - SOLRAPIQueueClient.aclReadersMap.clear(); - SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); SOLRAPIQueueClient.nodeMap.clear(); } diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java index a87a83b26..2037236fb 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java @@ -47,9 +47,9 @@ import org.json.JSONException; public class SOLRAPIQueueClient extends SOLRAPIClient { - public static List aclChangeSetQueue = Collections.synchronizedList(new ArrayList()); - public static Map> aclMap = Collections.synchronizedMap(new HashMap()); - public static Map aclReadersMap = Collections.synchronizedMap(new HashMap()); + protected static final List ACL_CHANGE_SET_QUEUE = Collections.synchronizedList(new ArrayList()); + protected static final Map> ACL_MAP = Collections.synchronizedMap(new HashMap()); + protected static final Map ACL_READERS_MAP = Collections.synchronizedMap(new HashMap()); public static List transactionQueue = Collections.synchronizedList(new ArrayList()); public static Map> nodeMap = Collections.synchronizedMap(new HashMap()); @@ -74,7 +74,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient throw new ConnectException("THROWING EXCEPTION, better be ready!"); } - int size = aclChangeSetQueue.size(); + int size = ACL_CHANGE_SET_QUEUE.size(); long maxTime = 0L; long maxId = 0L; @@ -83,7 +83,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient List aclChangeSetList = new ArrayList(); for(int i=0; i= minAclChangeSetId && aclChangeSet.getId() < maxAclChangeSetId) { aclChangeSetList.add(aclChangeSet); @@ -103,7 +103,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient for(int i=0; i allAcls = new ArrayList(); for(AclChangeSet aclChangeSet : aclChangeSets) { - List aclList = aclMap.get(aclChangeSet.getId()); + List aclList = ACL_MAP.get(aclChangeSet.getId()); allAcls.addAll(aclList); } return allAcls; @@ -170,7 +170,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient List allAclReaders = new ArrayList(); for(Acl acl : acls) { - AclReaders aclReaders = aclReadersMap.get(acl.getId()); + AclReaders aclReaders = ACL_READERS_MAP.get(acl.getId()); allAclReaders.add(aclReaders); } return allAclReaders; From 1852fec5711208f718ab1aebb31b4b1c2a6f0332 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Apr 2020 16:32:21 +0100 Subject: [PATCH 33/54] SEARCH-2202 Convert CoreDescriptorDecorator fields to constants. --- .../org/apache/solr/core/CoreDescriptorDecorator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/core/CoreDescriptorDecorator.java b/search-services/alfresco-search/src/main/java/org/apache/solr/core/CoreDescriptorDecorator.java index 848381ca1..3951e7b8b 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/core/CoreDescriptorDecorator.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/core/CoreDescriptorDecorator.java @@ -43,7 +43,7 @@ public class CoreDescriptorDecorator private static String SECURE_COMMS_PROPERTY = "alfresco.secureComms"; - public static ImmutableList substitutablePropertiesSecure = + public static final ImmutableList SUBSTITUTABLE_PROPERTIES_SECURE = ImmutableList.of( "alfresco.host", "alfresco.port", @@ -59,7 +59,7 @@ public class CoreDescriptorDecorator "alfresco.encryption.ssl.keystore.provider", "alfresco.encryption.ssl.truststore.type"); - public static ImmutableList substitutablePropertiesNone = + public static final ImmutableList SUBSTITUTABLE_PROPERTIES_NONE = ImmutableList.of( "alfresco.host", "alfresco.port", @@ -74,11 +74,11 @@ public class CoreDescriptorDecorator String comms = ConfigUtil.locateProperty(SECURE_COMMS_PROPERTY, "none"); if (comms.equals("https")) { - coreProperties = substitutablePropertiesSecure; + coreProperties = SUBSTITUTABLE_PROPERTIES_SECURE; } else { - coreProperties = substitutablePropertiesNone; + coreProperties = SUBSTITUTABLE_PROPERTIES_NONE; } try From 515a66f94454c0be66400166e80381d379ffd638 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Apr 2020 09:45:18 +0100 Subject: [PATCH 34/54] SEARCH-2202 Revert back to public constants in SOLRAPIQueueClient. --- .../java/org/alfresco/solr/client/SOLRAPIQueueClient.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java index 2037236fb..9471fba13 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java @@ -47,9 +47,9 @@ import org.json.JSONException; public class SOLRAPIQueueClient extends SOLRAPIClient { - protected static final List ACL_CHANGE_SET_QUEUE = Collections.synchronizedList(new ArrayList()); - protected static final Map> ACL_MAP = Collections.synchronizedMap(new HashMap()); - protected static final Map ACL_READERS_MAP = Collections.synchronizedMap(new HashMap()); + public static final List ACL_CHANGE_SET_QUEUE = Collections.synchronizedList(new ArrayList()); + public static final Map> ACL_MAP = Collections.synchronizedMap(new HashMap()); + public static final Map ACL_READERS_MAP = Collections.synchronizedMap(new HashMap()); public static List transactionQueue = Collections.synchronizedList(new ArrayList()); public static Map> nodeMap = Collections.synchronizedMap(new HashMap()); From 3943ee419224783e61784dca0aeba70079371d86 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 27 Apr 2020 12:03:27 +0100 Subject: [PATCH 35/54] SEARCH-2202 Remove unused class ResizableArrayList. Sonar was complaining about this class, but we don't actually use it, so deleting it. --- .../alfresco/solr/ResizeableArrayList.java | 168 ------------------ 1 file changed, 168 deletions(-) delete mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/ResizeableArrayList.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/ResizeableArrayList.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/ResizeableArrayList.java deleted file mode 100644 index f28ce84ce..000000000 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/ResizeableArrayList.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (C) 2005-2013 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr; - -import java.io.Serializable; -import java.util.AbstractList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.RandomAccess; - -/** - * A {@link List} implementation, backed by an array, taht supports resizing. - * - * This class supports the reuse of arrays across cache instances, reducing the number of - * temporary objects created by the Alfresco Solr indexing service. - * - * @author Alex Miller - */ -public class ResizeableArrayList extends AbstractList implements List, RandomAccess, Cloneable, Serializable -{ - private static final long serialVersionUID = 1L; - - private Object[] values; - private int size; - boolean active = false; - - public ResizeableArrayList() - { - this(10); - } - - public ResizeableArrayList(int initialSize) - { - values = new Object[initialSize]; - size = initialSize; - } - - @SuppressWarnings("unchecked") - @Override - public E get(int index) - { - checkSize(index); - return (E)values[index]; - } - - private void checkSize(int index) - { - if (index >= size) - { - throw new ArrayIndexOutOfBoundsException("Index: " + index + ", Size: " + values.length); - } - } - - /** - * Resize the underlying to at least minSize. - * - * @param minSize - * @throws IllegalStateException if the instance is not active. - */ - public void resize(int minSize) - { - isActive(); - int oldSize = values.length; - if (minSize > oldSize) - { - int newSize = (oldSize * 3)/2 + 1; - if (newSize < minSize) - { - newSize = minSize; - } - // minCapacity is usually close to size, so this is a win: - values = Arrays.copyOf(values, newSize); - } - size = minSize; - } - - @Override - public int size() - { - return size; - } - - /** - * Copy elements from from into this instance - */ - public void copyFrom(ResizeableArrayList from) - { - isActive(); - values = Arrays.copyOf(from.values, from.values.length); - size = from.size; - } - - @SuppressWarnings("unchecked") - @Override - public E set(int index, E value) - { - checkSize(index); - isActive(); - - E oldValue = (E)values[index]; - values[index] = value; - return oldValue; - } - - @SuppressWarnings("unchecked") - @Override - protected Object clone() throws CloneNotSupportedException - { - try { - ResizeableArrayList v = (ResizeableArrayList) super.clone(); - v.values = Arrays.copyOf(values, values.length); - v.size = size; - return v; - } catch (CloneNotSupportedException e) { - throw new InternalError(); - } - } - - private void isActive() - { - if (!active) - { - throw new IllegalStateException("Not active"); - } - } - - void activate() - { - active = true; - } - - void deactivate() - { - active = false; - for (int i = 0 ; i < size ; i++) - { - values[i] = null; - } - } - - /** - * Sort the elements, in-place, contained in this list using {@link Comparator#} - * - * @param comparator The {@link Comparator} to sort with - */ - @SuppressWarnings("unchecked") - public void sort(Comparator comparator) - { - DualPivotQuickSort.sort((E[])values, 0, size -1 , comparator); - } -} From 4b79c4b452cb06172bc6e3d760b3fc4903248a51 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 27 Apr 2020 12:07:47 +0100 Subject: [PATCH 36/54] SEARCH-2202 Simplify code in SolrInformationServer. Sonar complains about fall-through switch statements, but this logic can be much shorter anyway. --- .../org/alfresco/solr/SolrInformationServer.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index f8f617118..ba05c9e41 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -69,6 +69,9 @@ import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TXCOM import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TXID; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TYPE; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_VERSION; +import static org.alfresco.service.cmr.security.AuthorityType.EVERYONE; +import static org.alfresco.service.cmr.security.AuthorityType.GROUP; +import static org.alfresco.service.cmr.security.AuthorityType.GUEST; import static org.alfresco.solr.utils.Utils.notNullOrEmpty; import java.io.File; @@ -3216,17 +3219,10 @@ public class SolrInformationServer implements InformationServer */ private String addTenantToAuthority(String authority, String tenant) { - switch (AuthorityType.getAuthorityType(authority)) + AuthorityType authorityType = AuthorityType.getAuthorityType(authority); + if ((authorityType == GROUP || authorityType == EVERYONE || authorityType == GUEST) && !tenant.isEmpty()) { - case GROUP: - case EVERYONE: - case GUEST: - if (tenant.length() != 0) - { - return authority + "@" + tenant; - } - default: - break; + return authority + "@" + tenant; } return authority; } From dc91f1b3a3f5afe927c1dd4d07bc85a9c2b75565 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 27 Apr 2020 12:10:03 +0100 Subject: [PATCH 37/54] SEARCH-2202 Fix typo: hashcode -> hashCode. --- .../alfresco/solr/client/AlfrescoModel.java | 56 +++++++++-------- .../solr/client/AlfrescoModelDiff.java | 62 ++++++++++--------- 2 files changed, 61 insertions(+), 57 deletions(-) diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModel.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModel.java index faffa907d..a979d1094 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModel.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModel.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Solr Client - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Solr Client + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.solr.client; import org.alfresco.repo.dictionary.M2Model; @@ -52,7 +52,8 @@ public class AlfrescoModel { return checksum; } - + + @Override public boolean equals(Object other) { if (this == other) @@ -70,7 +71,8 @@ public class AlfrescoModel checksum.equals(model.getChecksum())); } - public int hashcode() + @Override + public int hashCode() { int result = 17; result = 31 * result + model.hashCode(); diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModelDiff.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModelDiff.java index 1b97a4445..ea9ae61d8 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModelDiff.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModelDiff.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Solr Client - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Solr Client + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.solr.client; import org.alfresco.service.namespace.QName; @@ -36,10 +36,10 @@ import org.alfresco.service.namespace.QName; */ public class AlfrescoModelDiff { - public static enum TYPE + public enum TYPE { NEW, CHANGED, REMOVED; - }; + } private QName modelName; private TYPE type; @@ -74,7 +74,8 @@ public class AlfrescoModelDiff { return newChecksum; } - + + @Override public boolean equals(Object other) { if(this == other) @@ -91,8 +92,9 @@ public class AlfrescoModelDiff diff.getOldChecksum().equals(getOldChecksum()) && diff.getNewChecksum().equals(getNewChecksum())); } - - public int hashcode() + + @Override + public int hashCode() { int result = 17; result = 31 * result + getModelName().hashCode(); From b136fff93985587cf908c628e95550fe30d052e0 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 27 Apr 2020 12:28:29 +0100 Subject: [PATCH 38/54] SEARCH-2202 The field analyzeRangeTerms is defined in the supertype. We're already setting the supertype field to true in the only constructor, so it's safe to use that instead. --- .../main/java/org/alfresco/solr/query/Solr4QueryParser.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java index b27ec4299..14587cf1f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java @@ -5496,8 +5496,6 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants throw new RuntimeException("Error analyzing multiTerm term: " + part, e); } } - - private boolean analyzeRangeTerms = true; protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) { final BytesRef start; @@ -5506,13 +5504,13 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants if (part1 == null) { start = null; } else { - start = analyzeRangeTerms ? analyzeMultitermTerm(field, part1) : new BytesRef(part1); + start = getAnalyzeRangeTerms() ? analyzeMultitermTerm(field, part1) : new BytesRef(part1); } if (part2 == null) { end = null; } else { - end = analyzeRangeTerms ? analyzeMultitermTerm(field, part2) : new BytesRef(part2); + end = getAnalyzeRangeTerms() ? analyzeMultitermTerm(field, part2) : new BytesRef(part2); } final TermRangeQuery query = new TermRangeQuery(field, start, end, startInclusive, endInclusive); From 4954eb2d2e0eb3073d04410e9e4de91ebe2d5dd5 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 27 Apr 2020 14:02:30 +0100 Subject: [PATCH 39/54] SEARCH-2202 Remove unused field. This exists in the superclass anyway. --- .../java/org/alfresco/solr/query/SolrContainerScorer.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/SolrContainerScorer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/SolrContainerScorer.java index 18affeca5..d59ff05bd 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/SolrContainerScorer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/SolrContainerScorer.java @@ -37,10 +37,6 @@ import org.apache.lucene.search.Weight; */ public class SolrContainerScorer extends Scorer { - // Unused - Weight weight; - - SolrContainerScorerDocIdSetIterator iterator; From d41d0ec9c4e5e98a645c03cc4ccbc4ec1dd8f2d5 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 27 Apr 2020 14:33:10 +0100 Subject: [PATCH 40/54] SEARCH-2202 Remove context field as it is present in supertype. --- .../org/alfresco/solr/transformer/CachedDocTransformer.java | 2 -- .../org/alfresco/solr/transformer/DocValueDocTransformer.java | 4 ---- 2 files changed, 6 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java index 5d6e230b6..c46b93314 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java @@ -47,8 +47,6 @@ public class CachedDocTransformer extends DocTransformer { protected final static Logger log = LoggerFactory.getLogger(CachedDocTransformer.class); - private ResultContext context; - /* (non-Javadoc) * @see org.apache.solr.response.transform.DocTransformer#getName() */ diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/DocValueDocTransformer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/DocValueDocTransformer.java index 0b3799a32..40586674f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/DocValueDocTransformer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/DocValueDocTransformer.java @@ -42,11 +42,7 @@ import org.slf4j.LoggerFactory; public class DocValueDocTransformer extends DocTransformer { protected final static Logger log = LoggerFactory.getLogger(DocValueDocTransformer.class); - - ResultContext context; - - /* (non-Javadoc) * @see org.apache.solr.response.transform.DocTransformer#getName() */ From 969cf3996238152b29a7850001d1326f9ac2bd8b Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 27 Apr 2020 14:37:12 +0100 Subject: [PATCH 41/54] SEARCH-2202 Make AdminHandler tests clearer. Sonar doesn't like tests with no assertions. --- .../org/alfresco/solr/AdminHandlerIT.java | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java index 140b6b1db..429ad4b01 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java @@ -1,8 +1,10 @@ package org.alfresco.solr; +import static org.junit.Assert.fail; + import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.common.SolrException; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.handler.admin.CoreAdminHandler; import org.apache.solr.response.SolrQueryResponse; @@ -32,19 +34,33 @@ public class AdminHandlerIT extends AbstractAlfrescoSolrIT } @Test - public void testhandledCores() throws Exception + public void testHandledCores() { - requestAction("newCore"); - requestAction("updateCore"); - requestAction("removeCore"); + try + { + requestAction("newCore"); + requestAction("updateCore"); + requestAction("removeCore"); + } + catch (Exception e) + { + fail("Expected to receive no exception, but got " + e); + } } @Test - public void testhandledReports() throws Exception + public void testHandledReports() { - requestAction("CHECK"); - requestAction("REPORT"); - requestAction("SUMMARY"); + try + { + requestAction("CHECK"); + requestAction("REPORT"); + requestAction("SUMMARY"); + } + catch (Exception e) + { + fail("Expected to receive no exception, but got " + e); + } } private void requestAction(String actionName) throws Exception { From b0bef746b882b2f244e4e07d47d9ef918a1b4cfb Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 27 Apr 2020 14:40:01 +0100 Subject: [PATCH 42/54] SEARCH-2202 Remove commented tests. It does not seem at all likely that these will ever be uncommented. They were commented out because they didn't work. --- .../java/org/alfresco/solr/CloudTest.java | 109 ------------------ 1 file changed, 109 deletions(-) delete mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/CloudTest.java diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CloudTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CloudTest.java deleted file mode 100644 index ab1e2bdf7..000000000 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CloudTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2005-2014 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr; - -public class CloudTest -{ - /* - public final static String QUERY = "a query"; - private Cloud cloud = new Cloud(); - @Mock SolrQueryRequest request; - - @SuppressWarnings({ "rawtypes" }) - @Test - public void testGetQueryWithZeroValues() - { - String fieldName = FIELD_DBID; - String operator = SolrInformationServer.AND; - Collection[] valueLists = new Collection[1]; - valueLists[0] = new ArrayList(); - String query = cloud.getQuery(fieldName, operator, valueLists); - assertEquals("", query); - - valueLists = new Collection[0]; - query = cloud.getQuery(fieldName, operator, valueLists); - assertEquals("", query); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test - public void testGetQueryWithOneValue() - { - String fieldName = FIELD_DBID; - String operator = SolrInformationServer.AND; - Collection[] valueLists = new Collection[1]; - valueLists[0] = new ArrayList(); - Object value = "value"; - valueLists[0].add(value); - String query = cloud.getQuery(fieldName, operator, valueLists); - assertEquals(FIELD_DBID + ":" + value, query); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test - public void testGetQueryWithManyValues() - { - String fieldName = FIELD_DBID; - String operator = SolrInformationServer.AND; - Collection[] valueLists = new Collection[2]; - valueLists[0] = new ArrayList(); - Object value1 = "value1"; - valueLists[0].add(value1); - Object value2 = "value2"; - valueLists[0].add(value2); - Object value3 = "value3"; - valueLists[0].add(value3); - valueLists[1] = new ArrayList(); - Object value4 = "value4"; - valueLists[1].add(value4); - String query = cloud.getQuery(fieldName, operator, valueLists); - assertEquals(FIELD_DBID + ":" + value1 + operator + FIELD_DBID + ":" + value2 + operator - + FIELD_DBID + ":" + value3 + operator + FIELD_DBID + ":" + value4, query); - } - - @Test - public void testExists() - { - boolean exists = cloud.exists(super.selectRequestHandler, request, QUERY); - assertFalse(exists); - } - - @Test - public void testGetDocList() - { - // The response is created in the getDocList method, and the aftsRequestHandler is simply mocked. - // Therefore nothing is expected to be on the response for tests, so this verifies behavior. - DocList docList = cloud.getDocList(super.aftsRequestHandler, request, QUERY); - assertNull(docList); - - ArgumentCaptor response = ArgumentCaptor.forClass(SolrQueryResponse.class); - verify(super.aftsRequestHandler).handleRequest(eq(request), response.capture()); - assertNotNull(response.getValue()); - } - - @Test - public void testSelect() - { - SolrParams params = new ModifiableSolrParams(request.getParams()); - ResultContext rc = cloud.getResultContext(super.aftsRequestHandler, request, params); - assertNull(rc); - } - */ - -} From 1aa959c186d15d9c59cc84361ea1dd493fa2d29b Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 27 Apr 2020 15:06:32 +0100 Subject: [PATCH 43/54] SEARCH-2202 Delete Solr unit tests. These are literally unit tests copied from the Solr project and which are testing Solr code. https://github.com/apache/lucene-solr/blob/master/lucene/analysis/common/src/test/org/apache/lucene/analysis/minhash/MinHashFilterTest.java --- .../analysis/minhash/MinHashFilterIT.java | 567 ------------------ 1 file changed, 567 deletions(-) delete mode 100644 search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java diff --git a/search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java b/search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java deleted file mode 100644 index c936e639d..000000000 --- a/search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java +++ /dev/null @@ -1,567 +0,0 @@ -package org.apache.lucene.analysis.minhash; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory; -import org.apache.lucene.analysis.minhash.MinHashFilter.FixedSizeTreeSet; -import org.apache.lucene.analysis.minhash.MinHashFilter.LongPair; -import org.apache.lucene.analysis.shingle.ShingleFilterFactory; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.util.CharFilterFactory; -import org.apache.lucene.analysis.util.TokenFilterFactory; -import org.apache.lucene.analysis.util.TokenizerFactory; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.automaton.CharacterRunAutomaton; -import org.apache.lucene.util.automaton.RegExp; -import org.junit.Test; - -public class MinHashFilterIT extends BaseTokenStreamTestCase -{ - @Test - public void testIntHash() { - LongPair hash = new LongPair(); - MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(0), 0, 4, 0, hash); - assertEquals(-3485513579396041028L, hash.val1); - assertEquals(6383328099726337777L, hash.val2); - } - - @Test - public void testStringHash() throws UnsupportedEncodingException { - LongPair hash = new LongPair(); - byte[] bytes = "woof woof woof woof woof".getBytes("UTF-16LE"); - MinHashFilter.murmurhash3_x64_128(bytes, 0, bytes.length, 0, hash); - assertEquals(7638079586852243959L, hash.val1); - assertEquals(4378804943379391304L, hash.val2); - } - - @Test - public void testSimpleOrder() throws UnsupportedEncodingException { - LongPair hash1 = new LongPair(); - hash1.val1 = 1; - hash1.val2 = 2; - LongPair hash2 = new LongPair(); - hash2.val1 = 2; - hash2.val2 = 1; - assert (hash1.compareTo(hash2) > 0); - } - - - @Test - public void testHashOrder() { - assertTrue(!MinHashFilter.isLessThanUnsigned(0l, 0l)); - assertTrue(MinHashFilter.isLessThanUnsigned(0l, -1l)); - assertTrue(MinHashFilter.isLessThanUnsigned(1l, -1l)); - assertTrue(MinHashFilter.isLessThanUnsigned(-2l, -1l)); - assertTrue(MinHashFilter.isLessThanUnsigned(1l, 2l)); - assertTrue(MinHashFilter.isLessThanUnsigned(Long.MAX_VALUE, Long.MIN_VALUE)); - - FixedSizeTreeSet minSet = new FixedSizeTreeSet(500); - HashSet unadded = new HashSet(); - for (int i = 0; i < 100; i++) { - LongPair hash = new LongPair(); - MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash); - LongPair peek = null; - if (minSet.size() > 0) { - peek = minSet.last(); - } - - if (!minSet.add(hash)) { - unadded.add(hash); - } else { - if (peek != null) { - if ((minSet.size() == 500) && !peek.equals(minSet.last())) { - unadded.add(peek); - } - } - } - } - assertEquals(100, minSet.size()); - assertEquals(0, unadded.size()); - - HashSet collisionDetection = new HashSet(); - unadded = new HashSet(); - minSet = new FixedSizeTreeSet(500); - for (int i = 0; i < 1000000; i++) { - LongPair hash = new LongPair(); - MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash); - collisionDetection.add(hash); - LongPair peek = null; - if (minSet.size() > 0) { - peek = minSet.last(); - } - - if (!minSet.add(hash)) { - unadded.add(hash); - } else { - if (peek != null) { - if ((minSet.size() == 500) && !peek.equals(minSet.last())) { - unadded.add(peek); - } - } - } - } - assertEquals(1000000, collisionDetection.size()); - assertEquals(500, minSet.size()); - assertEquals(999500, unadded.size()); - - LongPair last = null; - LongPair current = null; - while ((current = minSet.pollLast()) != null) { - if (last != null) { - assertTrue(isLessThan(current, last)); - } else { - - } - last = current; - } - } - - - - @Test - public void testHashNotRepeated() { - FixedSizeTreeSet minSet = new FixedSizeTreeSet(500); - HashSet unadded = new HashSet(); - for (int i = 0; i < 10000; i++) { - LongPair hash = new LongPair(); - MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash); - LongPair peek = null; - if (minSet.size() > 0) { - peek = minSet.last(); - } - if (!minSet.add(hash)) { - unadded.add(hash); - } else { - if (peek != null) { - if ((minSet.size() == 500) && !peek.equals(minSet.last())) { - unadded.add(peek); - } - } - } - } - assertEquals(500, minSet.size()); - - LongPair last = null; - LongPair current = null; - while ((current = minSet.pollLast()) != null) { - if (last != null) { - assertTrue(isLessThan(current, last)); - } else { - - } - last = current; - } - } - - @Test - public void testMockShingleTokenizer() throws IOException { - Tokenizer mockShingleTokenizer = createMockShingleTokenizer(5, - "woof woof woof woof woof" + " " + "woof woof woof woof puff"); - assertTokenStreamContents(mockShingleTokenizer, - new String[] {"woof woof woof woof woof", "woof woof woof woof puff"}); - } - - @Test - public void testTokenStreamSingleInput() throws IOException { - String[] hashes = new String[] {"℁팽徭聙↝ꇁ홱杯"}; - TokenStream ts = createTokenStream(5, "woof woof woof woof woof", 1, 1, 100, false); - assertTokenStreamContents(ts, hashes, new int[] {0}, - new int[] {24}, new String[] {MinHashFilter.MIN_HASH_TYPE}, new int[] {1}, new int[] {1}, 24, 0, null, - true); - - ts = createTokenStream(5, "woof woof woof woof woof", 2, 1, 1, false); - assertTokenStreamContents(ts, new String[] {new String(new char[] {0, 0, 8449, 54077, 64133, 32857, 8605, 41409}), - new String(new char[] {0, 1, 16887, 58164, 39536, 14926, 6529, 17276})}, new int[] {0, 0}, - new int[] {24, 24}, new String[] {MinHashFilter.MIN_HASH_TYPE, MinHashFilter.MIN_HASH_TYPE}, new int[] {1, 0}, new int[] {1, 1}, 24, 0, null, - true); - } - - @Test - public void testTokenStream1() throws IOException { - String[] hashes = new String[] {"℁팽徭聙↝ꇁ홱杯", - new String(new char[] {36347, 63457, 43013, 56843, 52284, 34231, 57934, 42302})}; - - TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 1, 1, 100,false); - assertTokenStreamContents(ts, hashes, new int[] {0, 0}, - new int[] {49, 49}, new String[] {MinHashFilter.MIN_HASH_TYPE, MinHashFilter.MIN_HASH_TYPE}, new int[] {1, 0}, - new int[] {1, 1}, 49, 0, null, true); - } - - private ArrayList getTokens(TokenStream ts) throws IOException { - ArrayList tokens = new ArrayList(); - ts.reset(); - while (ts.incrementToken()) { - CharTermAttribute termAttribute = ts.getAttribute(CharTermAttribute.class); - String token = new String(termAttribute.buffer(), 0, termAttribute.length()); - tokens.add(token); - } - ts.end(); - ts.close(); - - return tokens; - } - - private ArrayList getTokens(Analyzer analyzer, String field, String value) throws IOException - { - ArrayList tokens = new ArrayList(); - - TokenStream ts = analyzer.tokenStream(field, value); - ts.reset(); - while(ts.incrementToken()) - { - CharTermAttribute termAttribute = ts.getAttribute(CharTermAttribute.class); - String token = new String(termAttribute.buffer(), 0, termAttribute.length()); - tokens.add(token); - } - ts.end(); - ts.close(); - - return tokens; - } - - @Test - public void testTokenStream2() throws IOException { - TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 100, 1, 1, false); - ArrayList tokens = getTokens(ts); - ts.close(); - - assertEquals(100, tokens.size()); - } - - @Test - public void testTokenStream3() throws IOException { - TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 1, 10, false); - ArrayList tokens = getTokens(ts); - ts.close(); - - assertEquals(20, tokens.size()); - } - - @Test - public void testTokenStream4() throws IOException { - TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 10, 1, false); - ArrayList tokens = getTokens(ts); - ts.close(); - - assertEquals(20, tokens.size()); - - ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 10, 1, true); - tokens = getTokens(ts); - ts.close(); - - assertEquals(100, tokens.size()); - - } - - @Test - public void testLSHQuery() throws IOException - { - Analyzer analyzer = createMinHashAnalyzer(5, 1, 100); - IndexWriterConfig config = new IndexWriterConfig(analyzer); - - RAMDirectory directory = new RAMDirectory(); - IndexWriter writer = new IndexWriter(directory, config); - Document doc = new Document(); - doc.add(new TextField("text", "woof woof woof woof woof", Store.NO)); - writer.addDocument(doc); - - doc = new Document(); - doc.add(new TextField("text", "woof woof woof woof woof puff", Store.NO)); - writer.addDocument(doc); - - doc = new Document(); - doc.add(new TextField("text", "woof woof woof woof puff", Store.NO)); - writer.addDocument(doc); - - writer.commit(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory)); - - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", "℁팽徭聙↝ꇁ홱杯"))), Occur.SHOULD); - builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", new String(new char[] {36347, 63457, 43013, 56843, 52284, 34231, 57934, 42302})))), Occur.SHOULD); - builder.setDisableCoord(true); - TopDocs topDocs = searcher.search(builder.build(), 10); - - assertEquals(3, topDocs.totalHits); - - float score = topDocs.scoreDocs[0].score; - assertEquals(topDocs.scoreDocs[1].score, score/2, 0f); - assertEquals(topDocs.scoreDocs[2].score, score/2, 0f); - - } - - - - @Test - public void testLSHQuery2() throws IOException - { - String[] parts = new String[]{"one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"}; - int min = 5; - - Analyzer analyzer = createMinHashAnalyzer(min, 1, 100); - IndexWriterConfig config = new IndexWriterConfig(analyzer); - - RAMDirectory directory = new RAMDirectory(); - IndexWriter writer = new IndexWriter(directory, config); - - for(int i = 0; i < parts.length; i++) - { - StringBuilder builder = new StringBuilder(); - for(int j = 0; j < parts.length -i; j++) - { - if(builder.length() > 0) - { - builder.append(" "); - } - builder.append(parts[i+j]); - if(j >= min -1) - { - Document doc = new Document(); - doc.add(new TextField("text", builder.toString(), Store.NO)); - writer.addDocument(doc); - } - } - } - - writer.commit(); - writer.close(); - - - IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory)); - - TopDocs topDocs = searcher.search(buildQuery("text", "one two three four five", min, 1, 100), 100); - assertEquals(6, topDocs.totalHits); - assertAllScores(topDocs, 1.0f); - topDocs = searcher.search(buildQuery("text", "two three four five six", min, 1, 100), 100); - assertEquals(10, topDocs.totalHits); - assertAllScores(topDocs, 1.0f); - topDocs = searcher.search(buildQuery("text", "three four five six seven", min, 1, 100), 100); - assertEquals(12, topDocs.totalHits); - assertAllScores(topDocs, 1.0f); - topDocs = searcher.search(buildQuery("text", "four five six seven eight", min, 1, 100), 100); - assertEquals(12, topDocs.totalHits); - assertAllScores(topDocs, 1.0f); - topDocs = searcher.search(buildQuery("text", "five six seven eight nine", min, 1, 100), 100); - assertEquals(10, topDocs.totalHits); - assertAllScores(topDocs, 1.0f); - topDocs = searcher.search(buildQuery("text", "six seven eight nine ten", min, 1, 100), 100); - assertEquals(6, topDocs.totalHits); - assertAllScores(topDocs, 1.0f); - - topDocs = searcher.search(buildQuery("text", "one two three four five six", min, 1, 100), 100); - assertEquals(11, topDocs.totalHits); - - topDocs = searcher.search(buildQuery("text", "one two three four five six seven eight nine ten", min, 1, 100), 100); - assertEquals(21, topDocs.totalHits); - for(int i = 0; i < topDocs.totalHits; i++) - { - System.out.println(i+" = "+topDocs.scoreDocs[i]); - } - - float topScore = 6.0f; - assertEquals(topDocs.scoreDocs[0].score, topScore, 0.001f); - assertEquals(topDocs.scoreDocs[1].score, topScore * 5/6, 0.001f); - assertEquals(topDocs.scoreDocs[2].score, topScore * 5/6, 0.001f); - assertEquals(topDocs.scoreDocs[3].score, topScore * 4/6, 0.001f); - assertEquals(topDocs.scoreDocs[4].score, topScore * 4/6, 0.001f); - assertEquals(topDocs.scoreDocs[5].score, topScore * 4/6, 0.001f); - assertEquals(topDocs.scoreDocs[6].score, topScore * 3/6, 0.001f); - assertEquals(topDocs.scoreDocs[7].score, topScore * 3/6, 0.001f); - assertEquals(topDocs.scoreDocs[8].score, topScore * 3/6, 0.001f); - assertEquals(topDocs.scoreDocs[9].score, topScore * 3/6, 0.001f); - assertEquals(topDocs.scoreDocs[10].score, topScore * 2/6, 0.001f); - assertEquals(topDocs.scoreDocs[11].score, topScore * 2/6, 0.001f); - assertEquals(topDocs.scoreDocs[12].score, topScore * 2/6, 0.001f); - assertEquals(topDocs.scoreDocs[13].score, topScore * 2/6, 0.001f); - assertEquals(topDocs.scoreDocs[14].score, topScore * 2/6, 0.001f); - assertEquals(topDocs.scoreDocs[15].score, topScore * 1/6, 0.001f); - assertEquals(topDocs.scoreDocs[16].score, topScore * 1/6, 0.001f); - assertEquals(topDocs.scoreDocs[17].score, topScore * 1/6, 0.001f); - assertEquals(topDocs.scoreDocs[18].score, topScore * 1/6, 0.001f); - assertEquals(topDocs.scoreDocs[19].score, topScore * 1/6, 0.001f); - assertEquals(topDocs.scoreDocs[20].score, topScore * 1/6, 0.001f); - - } - - - private void assertAllScores(TopDocs topDocs, float score) - { - for(int i = 0; i < topDocs.totalHits; i++) - { - assertEquals(topDocs.scoreDocs[i].score, score, 0f); - } - } - - private Query buildQuery(String field, String query, int min, int hashCount, int hashSetSize) throws IOException - { - TokenizerChain chain = createMinHashAnalyzer(min, hashCount, hashSetSize); - ArrayList tokens = getTokens(chain, field, query); - chain.close(); - - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - for(String token : tokens) - { - builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", token))), Occur.SHOULD); - } - builder.setDisableCoord(true); - return builder.build(); - } - - public static TokenStream createTokenStream(int shingleSize, String shingles, int hashCount, int bucketCount, int hashSetSize, boolean withRotation) { - Tokenizer tokenizer = createMockShingleTokenizer(shingleSize, shingles); - HashMap lshffargs = new HashMap(); - lshffargs.put("hashCount", "" + hashCount); - lshffargs.put("bucketCount", "" + bucketCount); - lshffargs.put("hashSetSize", "" + hashSetSize); - lshffargs.put("withRotation", "" + withRotation); - MinHashFilterFactory lshff = new MinHashFilterFactory(lshffargs); - return lshff.create(tokenizer); - } - - public static TokenizerChain createMinHashAnalyzer(int min, int hashCount, int hashSetSize) - { - WhitespaceTokenizerFactory icutf = new WhitespaceTokenizerFactory(Collections.emptyMap()); - HashMap sffargs = new HashMap(); - sffargs.put("minShingleSize", ""+min); - sffargs.put("maxShingleSize", ""+min); - sffargs.put("outputUnigrams", "false"); - sffargs.put("outputUnigramsIfNoShingles", "false"); - sffargs.put("tokenSeparator", " "); - ShingleFilterFactory sff = new ShingleFilterFactory(sffargs); - HashMap lshffargs = new HashMap(); - lshffargs.put("hashCount", ""+hashCount); - lshffargs.put("hashSetSize", ""+hashSetSize); - MinHashFilterFactory lshff = new MinHashFilterFactory(lshffargs); - - TokenizerChain chain = new TokenizerChain(new CharFilterFactory[]{}, icutf, new TokenFilterFactory[]{sff, lshff}); - return chain; - } - - public static Tokenizer createMockShingleTokenizer(int shingleSize, String shingles) { - MockTokenizer tokenizer = new MockTokenizer( - new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+([ \t\r\n]+[^ \t\r\n]+){4}").toAutomaton()), - true); - tokenizer.setEnableChecks(true); - if (shingles != null) { - tokenizer.setReader(new StringReader(shingles)); - } - return tokenizer; - } - - private boolean isLessThan(LongPair hash1, LongPair hash2) { - if (MinHashFilter.isLessThanUnsigned(hash1.val2, hash2.val2)) { - return true; - } else if (hash1.val2 == hash2.val2) { - return (MinHashFilter.isLessThanUnsigned(hash1.val1, hash2.val1)); - } else { - return false; - } - } - - - /** - * An analyzer that uses a tokenizer and a list of token filters to - * create a TokenStream - lifted from SOLR to make this analyzer test lucene only. - */ - public static class TokenizerChain extends Analyzer { - - final private CharFilterFactory[] charFilters; - final private TokenizerFactory tokenizer; - final private TokenFilterFactory[] filters; - - - /** - * Creates a new TokenizerChain. - * - * @param charFilters Factories for the CharFilters to use, if any - if null, will be treated as if empty. - * @param tokenizer Factory for the Tokenizer to use, must not be null. - * @param filters Factories for the TokenFilters to use if any- if null, will be treated as if empty. - */ - public TokenizerChain(CharFilterFactory[] charFilters, TokenizerFactory tokenizer, TokenFilterFactory[] filters) { - this.charFilters = charFilters; - this.tokenizer = tokenizer; - this.filters = filters; - } - - @Override - public Reader initReader(String fieldName, Reader reader) { - if (charFilters != null && charFilters.length > 0) { - Reader cs = reader; - for (CharFilterFactory charFilter : charFilters) { - cs = charFilter.create(cs); - } - reader = cs; - } - return reader; - } - - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tk = tokenizer.create(); - TokenStream ts = tk; - for (TokenFilterFactory filter : filters) { - ts = filter.create(ts); - } - return new TokenStreamComponents(tk, ts); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("TokenizerChain("); - for (CharFilterFactory filter: charFilters) { - sb.append(filter); - sb.append(", "); - } - sb.append(tokenizer); - for (TokenFilterFactory filter: filters) { - sb.append(", "); - sb.append(filter); - } - sb.append(')'); - return sb.toString(); - } - } -} From 37c57d1d521928d778d85eb50e56e27b560f1c5d Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 28 Apr 2020 10:28:42 +0100 Subject: [PATCH 44/54] SEARCH-2202 Handle case where shard range is missed. --- .../solr/tracker/DocRouterFactory.java | 14 +++-- .../solr/tracker/DocRouterFactoryTest.java | 60 +++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DocRouterFactoryTest.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java index 672313371..21d3b38ac 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java @@ -18,6 +18,7 @@ */ package org.alfresco.solr.tracker; +import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.repo.index.shard.ShardMethodEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,14 +45,15 @@ public class DocRouterFactory LOGGER.info("Sharding via DB_ID"); return new DBIDRouter(); case DB_ID_RANGE: - if(properties.containsKey(SHARD_RANGE_KEY)) + if(!properties.containsKey(SHARD_RANGE_KEY)) { - LOGGER.info("Sharding via DB_ID_RANGE"); - String[] pair =properties.getProperty(SHARD_RANGE_KEY).split("-"); - long start = Long.parseLong(pair[0]); - long end = Long.parseLong(pair[1]); - return new DBIDRangeRouter(start, end); + throw new AlfrescoRuntimeException("DB_ID_RANGE sharding requires the " + SHARD_RANGE_KEY + " property to be set."); } + LOGGER.info("Sharding via DB_ID_RANGE"); + String[] pair = properties.getProperty(SHARD_RANGE_KEY).split("-"); + long start = Long.parseLong(pair[0]); + long end = Long.parseLong(pair[1]); + return new DBIDRangeRouter(start, end); case ACL_ID: LOGGER.info("Sharding via ACL_ID"); return new ACLIDMurmurRouter(); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DocRouterFactoryTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DocRouterFactoryTest.java new file mode 100644 index 000000000..e4236c7b9 --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DocRouterFactoryTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2005-2014 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.tracker; + +import static org.alfresco.solr.tracker.DocRouterFactory.SHARD_RANGE_KEY; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Properties; + +import org.alfresco.error.AlfrescoRuntimeException; +import org.alfresco.repo.index.shard.ShardMethodEnum; +import org.junit.Test; + +/** Unit tests for the {@link DocRouterFactory}. */ +public class DocRouterFactoryTest +{ + /** Check that a DB_ID_RANGE router can be created. */ + @Test + public void testDBIDRANGEWithShardRangeKey() + { + Properties mockProperties = mock(Properties.class); + when(mockProperties.containsKey(SHARD_RANGE_KEY)).thenReturn(true); + when(mockProperties.getProperty(SHARD_RANGE_KEY)).thenReturn("100000000-150000000"); + + // Call the method under test. + DocRouter docRouter = DocRouterFactory.getRouter(mockProperties, ShardMethodEnum.DB_ID_RANGE); + + assertTrue("Expected to get a DBIDRangeRouter.", docRouter instanceof DBIDRangeRouter); + DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter) docRouter; + assertEquals("Unexpected start of range.", dbidRangeRouter.getStartRange(), 100000000L); + assertEquals("Unexpected end of range.", dbidRangeRouter.getEndRange(), 150000000L); + } + + /** Check that an exception is raised if the range information is missing. */ + @Test(expected = AlfrescoRuntimeException.class) + public void testDBIDRANGEWithoutShardRangeKey() + { + Properties mockProperties = mock(Properties.class); + DocRouterFactory.getRouter(mockProperties, ShardMethodEnum.DB_ID_RANGE); + } +} From 73ac9e44aa31f6c875ff6ca1801a5c6b2a8ad763 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 28 Apr 2020 10:51:38 +0100 Subject: [PATCH 45/54] SEARCH-2202 Ensure each class has its own logger. Make loggers static and final, and use slf4j tokenisation. --- .../solr/tracker/AbstractTracker.java | 20 +++--- .../alfresco/solr/tracker/CascadeTracker.java | 67 +++++++++---------- .../solr/tracker/CoreStatePublisher.java | 9 ++- .../alfresco/solr/tracker/ModelTracker.java | 11 +-- .../solr/tracker/SlaveCoreStatePublisher.java | 6 +- 5 files changed, 59 insertions(+), 54 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java index d0a19474f..038f56664 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java @@ -43,8 +43,7 @@ public abstract class AbstractTracker implements Tracker static final long TIME_STEP_32_DAYS_IN_MS = 1000 * 60 * 60 * 24 * 32L; static final long TIME_STEP_1_HR_IN_MS = 60 * 60 * 1000L; static final String SHARD_METHOD_DBID = "DB_ID"; - - protected final Logger logger = LoggerFactory.getLogger(getClass()); + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractTracker.class); protected Properties props; protected SOLRAPIClient client; @@ -170,7 +169,7 @@ public abstract class AbstractTracker implements Tracker if(runLock.availablePermits() == 0) { - logger.info("[{} / {} / {}] Tracker already registered.", coreName, trackerId, iterationId); + LOGGER.info("[{} / {} / {}] Tracker already registered.", coreName, trackerId, iterationId); return; } @@ -191,7 +190,7 @@ public abstract class AbstractTracker implements Tracker { this.state = getTrackerState(); - logger.debug("[{} / {} / {}] Global Tracker State set to: {}", coreName, trackerId, iterationId, this.state.toString()); + LOGGER.debug("[{} / {} / {}] Global Tracker State set to: {}", coreName, trackerId, iterationId, this.state.toString()); this.state.setRunning(true); } else @@ -209,28 +208,25 @@ public abstract class AbstractTracker implements Tracker catch(IndexTrackingShutdownException t) { setRollback(true, t); - logger.info("[{} / {} / {}] Tracking cycle stopped. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); + LOGGER.info("[{} / {} / {}] Tracking cycle stopped. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); } catch(Throwable t) { setRollback(true, t); if (t instanceof SocketTimeoutException || t instanceof ConnectException) { - logger.warn("[{} / {} / {}] Tracking communication timed out. See the stacktrace below for further details.", coreName, trackerId, iterationId); - if (logger.isDebugEnabled()) - { - logger.debug("[{} / {} / {}] Stack trace", coreName, trackerId, iterationId, t); - } + LOGGER.warn("[{} / {} / {}] Tracking communication timed out. See the stacktrace below for further details.", coreName, trackerId, iterationId); + LOGGER.debug("[{} / {} / {}] Stack trace", coreName, trackerId, iterationId, t); } else { - logger.error("[{} / {} / {}] Tracking failure. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); + LOGGER.error("[{} / {} / {}] Tracking failure. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); } } } catch (InterruptedException e) { - logger.error("[{} / {} / {}] Semaphore interruption. See the stacktrace below for further details.", coreName, trackerId, iterationId, e); + LOGGER.error("[{} / {} / {}] Semaphore interruption. See the stacktrace below for further details.", coreName, trackerId, iterationId, e); } finally { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java index 6396500d4..076abe576 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java @@ -18,6 +18,10 @@ */ package org.alfresco.solr.tracker; +import static java.util.stream.Collectors.joining; + +import static org.alfresco.solr.utils.Utils.notNullOrEmpty; + import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; @@ -31,24 +35,17 @@ import org.alfresco.solr.InformationServer; import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.client.Transaction; -import org.apache.commons.codec.EncoderException; import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static java.util.stream.Collectors.joining; -import static org.alfresco.solr.utils.Utils.notNullOrEmpty; - /* * This tracks Cascading Updates * @author Joel Bernstein */ public class CascadeTracker extends AbstractTracker implements Tracker { - - protected final static Logger log = LoggerFactory.getLogger(CascadeTracker.class); - - + private static final Logger LOGGER = LoggerFactory.getLogger(CascadeTracker.class); public CascadeTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) @@ -64,7 +61,7 @@ public class CascadeTracker extends AbstractTracker implements Tracker } @Override - protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException, EncoderException + protected void doTrack(String iterationId) throws IOException, JSONException { // MetadataTracker must wait until ModelTracker has run ModelTracker modelTracker = this.infoSrv.getAdminHandler().getTrackerRegistry().getModelTracker(); @@ -74,15 +71,15 @@ public class CascadeTracker extends AbstractTracker implements Tracker } } - public void maintenance() throws Exception { - + public void maintenance() + { } public boolean hasMaintenance() { return false; } - private void trackRepository(String iterationId) throws IOException, AuthenticationException, JSONException, EncoderException + private void trackRepository(String iterationId) throws IOException, JSONException { checkShutdown(); processCascades(iterationId); @@ -123,7 +120,8 @@ public class CascadeTracker extends AbstractTracker implements Tracker } } - public void invalidateState() { + public void invalidateState() + { super.invalidateState(); infoSrv.setCleanCascadeTxnFloor(-1); } @@ -131,55 +129,58 @@ public class CascadeTracker extends AbstractTracker implements Tracker private void processCascades(String iterationId) throws IOException { int num = 50; - List txBatch = null; - do { - try { + List txBatch; + do + { + try + { getWriteLock().acquire(); txBatch = infoSrv.getCascades(num); - if(txBatch.size() == 0) { + if(txBatch.size() == 0) + { //No transactions to process for cascades. return; } - ArrayList txIds = new ArrayList(); - Set txIdSet = new HashSet(); - for (Transaction tx : txBatch) { + ArrayList txIds = new ArrayList<>(); + Set txIdSet = new HashSet<>(); + for (Transaction tx : txBatch) + { txIds.add(tx.getId()); txIdSet.add(tx.getId()); } List nodeMetaDatas = infoSrv.getCascadeNodes(txIds); - //System.out.println("########### Cascade node meta datas:"+nodeMetaDatas.size()); - if(nodeMetaDatas.size() > 0) { - LinkedList stack = new LinkedList(); + if(nodeMetaDatas.size() > 0) + { + LinkedList stack = new LinkedList<>(); stack.addAll(nodeMetaDatas); int batchSize = 10; - do { - List batch = new ArrayList(); - while (batch.size() < batchSize && stack.size() > 0) { + do + { + List batch = new ArrayList<>(); + while (batch.size() < batchSize && stack.size() > 0) + { batch.add(stack.removeFirst()); } - CascadeIndexWorkerRunnable worker = new CascadeIndexWorkerRunnable(this.threadHandler, batch, infoSrv); - if (logger.isTraceEnabled()) + if (LOGGER.isTraceEnabled()) { String nodes = notNullOrEmpty(batch).stream() .map(NodeMetaData::getId) .map(Object::toString) .collect(joining(",")); - logger.trace("[{} / {} / {} / {}] Worker has been created for nodes {}", coreName, trackerId, iterationId, worker.hashCode(), nodes); + LOGGER.trace("[{} / {} / {} / {}] Worker has been created for nodes {}", coreName, trackerId, iterationId, worker.hashCode(), nodes); } this.threadHandler.scheduleTask(worker); - } - while (stack.size() > 0); + } while (stack.size() > 0); } //Update the transaction records. updateTransactionsAfterAsynchronous(txBatch); - //System.out.println("######################: Finished Cascade Run #########"); } catch (AuthenticationException e) { @@ -195,10 +196,8 @@ public class CascadeTracker extends AbstractTracker implements Tracker } finally { - //System.out.println("###################: Releasing Cascade write lock"); getWriteLock().release(); } - } while(txBatch.size() > 0); } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java index aec77956c..45352c1e9 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java @@ -46,6 +46,8 @@ import org.alfresco.solr.NodeReport; import org.alfresco.solr.TrackerState; import org.alfresco.solr.client.SOLRAPIClient; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashMap; import java.util.Optional; @@ -63,6 +65,7 @@ import java.util.Properties; */ public abstract class CoreStatePublisher extends AbstractTracker { + private static final Logger LOGGER = LoggerFactory.getLogger(CoreStatePublisher.class); DocRouter docRouter; private final boolean isMaster; @@ -118,7 +121,7 @@ public abstract class CoreStatePublisher extends AbstractTracker updateShardProperty(); if (shardProperty.isEmpty()) { - logger.warn("Sharding property {} was set to {}, but no such property was found.", SHARD_KEY_KEY, shardKeyName); + LOGGER.warn("Sharding property {} was set to {}, but no such property was found.", SHARD_KEY_KEY, shardKeyName); } }); } @@ -134,11 +137,11 @@ public abstract class CoreStatePublisher extends AbstractTracker { if (updatedShardProperty.isEmpty()) { - logger.warn("The model defining {} property has been disabled", shardKeyName); + LOGGER.warn("The model defining {} property has been disabled", shardKeyName); } else { - logger.info("New {} property found for {} ", SHARD_KEY_KEY, shardKeyName); + LOGGER.info("New {} property found for {}", SHARD_KEY_KEY, shardKeyName); } } shardProperty = updatedShardProperty; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java index 946634c3c..7b1223be7 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java @@ -47,6 +47,8 @@ import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.config.ConfigUtil; import org.apache.solr.core.SolrResourceLoader; import org.json.JSONException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @startuml @@ -82,6 +84,7 @@ import org.json.JSONException; */ public class ModelTracker extends AbstractTracker implements Tracker { + private static final Logger LOGGER = LoggerFactory.getLogger(ModelTracker.class); private final Set indexedStores = new HashSet<>(); private final Set ignoredStores = new HashSet<>(); @@ -106,7 +109,7 @@ public class ModelTracker extends AbstractTracker implements Tracker super(p, client, coreName, informationServer, Tracker.Type.MODEL); String normalSolrHome = SolrResourceLoader.normalizeDir(solrHome); alfrescoModelDir = new File(ConfigUtil.locateProperty("solr.model.dir", normalSolrHome+"alfrescoModels")); - logger.info("Alfresco Model dir " + alfrescoModelDir); + LOGGER.info("Alfresco Model dir {}", alfrescoModelDir); if (!alfrescoModelDir.exists()) { alfrescoModelDir.mkdir(); @@ -197,7 +200,7 @@ public class ModelTracker extends AbstractTracker implements Tracker int registeredSearcherCount = this.infoSrv.getRegisteredSearcherCount(); if (registeredSearcherCount >= getMaxLiveSearchers()) { - logger.info(".... skipping tracking registered searcher count = " + registeredSearcherCount); + LOGGER.info(".... skipping tracking registered searcher count = {}", registeredSearcherCount); return; } @@ -268,7 +271,7 @@ public class ModelTracker extends AbstractTracker implements Tracker } catch (Throwable t) { - logger.error("Model tracking failed for core: "+ coreName, t); + LOGGER.error("Model tracking failed for core: {}", coreName, t); } } @@ -534,7 +537,7 @@ public class ModelTracker extends AbstractTracker implements Tracker { loadedModels.add(modelName); } - logger.info("Loading model " + model.getName()); + LOGGER.info("Loading model {}", model.getName()); } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java index 39e4ff3d2..5562c34d6 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java @@ -26,6 +26,8 @@ import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.TrackerState; import org.alfresco.solr.client.SOLRAPIClient; import org.apache.commons.codec.EncoderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Properties; @@ -47,6 +49,8 @@ import java.util.Properties; */ public class SlaveCoreStatePublisher extends CoreStatePublisher { + private static final Logger LOGGER = LoggerFactory.getLogger(SlaveCoreStatePublisher.class); + public SlaveCoreStatePublisher( boolean isMaster, Properties coreProperties, @@ -67,7 +71,7 @@ public class SlaveCoreStatePublisher extends CoreStatePublisher } catch (EncoderException | IOException | AuthenticationException exception ) { - logger.error("Unable to publish this node state. " + + LOGGER.error("Unable to publish this node state. " + "A failure condition has been met during the outbound subscription message encoding process. " + "See the stacktrace below for further details.", exception); } From e2e6736679e11e59745e4af24b2bd35125bf41a8 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Tue, 28 Apr 2020 13:53:02 +0100 Subject: [PATCH 46/54] SEARCH-2219 Update base Java image --- search-services/packaging/src/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/packaging/src/docker/Dockerfile b/search-services/packaging/src/docker/Dockerfile index c7cb706e0..89462d717 100644 --- a/search-services/packaging/src/docker/Dockerfile +++ b/search-services/packaging/src/docker/Dockerfile @@ -1,6 +1,6 @@ # Alfresco Search Services ${project.version} Docker Image -FROM alfresco/alfresco-base-java:11.0.1-openjdk-centos-7-72b88c6f1f4c +FROM alfresco/alfresco-base-java:11.0.7-openjdk-centos-7-145f5d28c8ca LABEL creator="Gethin James" maintainer="Alfresco Search Services Team" ENV DIST_DIR /opt/alfresco-search-services From 9a31f984e9732f023a661890d9f4ef8ea5088349 Mon Sep 17 00:00:00 2001 From: Keerat Date: Wed, 29 Apr 2020 13:38:13 +0100 Subject: [PATCH 47/54] SEARCH-2121 Adding test group to failing tests with cascade tracking disabled --- .../searchServices/cmis/SolrSearchByAspectTests.java | 3 ++- .../searchServices/cmis/SolrSearchByIdTests.java | 3 ++- .../searchServices/cmis/SolrSearchByPathTests.java | 3 ++- .../searchServices/cmis/SolrSearchInTreeTests.java | 3 ++- .../searchServices/search/FacetFieldsSearchTest.java | 8 +++++--- .../searchServices/search/FacetRangeSearchTest.java | 6 +++--- .../searchServices/search/FacetedSearchTest.java | 2 +- .../searchServices/search/PivotFacetedSearchTest.java | 2 +- .../searchServices/search/SearchAFTSInFieldTest.java | 2 +- .../functional/searchServices/search/SearchAPATHTest.java | 7 ++++--- .../search/SearchSecondaryAssociationTest.java | 3 ++- .../functional/searchServices/search/SearchTest.java | 4 ++-- .../functional/searchServices/search/StatsSearchTest.java | 2 +- .../search/tracker/CascadingIntegrationTest.java | 4 ++-- 14 files changed, 30 insertions(+), 22 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java index 5228c38e6..5c073dd76 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java @@ -1,5 +1,6 @@ package org.alfresco.test.search.functional.searchServices.cmis; +import org.alfresco.search.TestGroup; import org.alfresco.utility.Utility; import org.alfresco.utility.data.CustomObjectTypeProperties; import org.alfresco.utility.data.provider.XMLDataConfig; @@ -128,7 +129,7 @@ public class SolrSearchByAspectTests extends AbstractCmisE2ETest Utility.waitToLoopTime(getSolrWaitTimeInSeconds()); } - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) @XMLDataConfig(file = "src/test/resources/testdata/search-by-aspect.xml") public void executeSearchByAspect(QueryModel query) throws Exception { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java index 8d10955fd..1db07b09b 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java @@ -1,5 +1,6 @@ package org.alfresco.test.search.functional.searchServices.cmis; +import org.alfresco.search.TestGroup; import org.alfresco.utility.Utility; import org.alfresco.utility.data.CustomObjectTypeProperties; import org.alfresco.utility.data.provider.XMLDataConfig; @@ -86,7 +87,7 @@ public class SolrSearchByIdTests extends AbstractCmisE2ETest Utility.waitToLoopTime(getSolrWaitTimeInSeconds()); } - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) @XMLDataConfig(file = "src/test/resources/testdata/search-by-id.xml") public void executeSearchById(QueryModel query) throws Exception { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java index 1c86b2912..c81373894 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java @@ -1,5 +1,6 @@ package org.alfresco.test.search.functional.searchServices.cmis; +import org.alfresco.search.TestGroup; import org.alfresco.utility.Utility; import org.alfresco.utility.data.provider.XMLDataConfig; import org.alfresco.utility.data.provider.XMLTestData; @@ -48,7 +49,7 @@ public class SolrSearchByPathTests extends AbstractCmisE2ETest Utility.waitToLoopTime(getSolrWaitTimeInSeconds()); } - @Test(dependsOnMethods = "prepareDataForSearchByPath", dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") + @Test(dependsOnMethods = "prepareDataForSearchByPath", dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) @XMLDataConfig(file = "src/test/resources/testdata/search-by-path.xml") public void executeSearchByPathQueries(QueryModel query) { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java index cf6647e86..c279ef57d 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java @@ -1,5 +1,6 @@ package org.alfresco.test.search.functional.searchServices.cmis; +import org.alfresco.search.TestGroup; import org.alfresco.utility.Utility; import org.alfresco.utility.data.provider.XMLDataConfig; import org.alfresco.utility.data.provider.XMLTestDataProvider; @@ -55,7 +56,7 @@ public class SolrSearchInTreeTests extends AbstractCmisE2ETest dataContent.deleteSite(testSite); } - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) @XMLDataConfig(file = "src/test/resources/testdata/search-in-tree.xml") public void executeCMISQuery(QueryModel query) throws Exception { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java index 50e75cc67..597722227 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java @@ -31,8 +31,10 @@ import org.alfresco.rest.search.RestRequestQueryModel; import org.alfresco.rest.search.RestResultBucketsModel; import org.alfresco.rest.search.SearchRequest; import org.alfresco.rest.search.SearchResponse; +import org.alfresco.search.TestGroup; import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; import org.alfresco.utility.data.RandomData; +import org.alfresco.utility.data.provider.XMLTestDataProvider; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FileType; import org.alfresco.utility.model.FolderModel; @@ -93,7 +95,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest waitForContentIndexing(htmlFile.getContent(), true); } - @Test + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) public void testSearchFacetFieldsBucketExcludedWhenMinCount2() throws Exception { // Create Query with FacetFields: Site and Content MimeType @@ -129,7 +131,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest } - @Test + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) public void testSearchWithFacetFieldsMinCountChecks() throws Exception { SearchRequest query = new SearchRequest(); @@ -204,7 +206,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest Assert.assertNull(response.getContext().getFacetsFields()); } - @Test + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) public void testSearchWithFacetFieldsOnlyFacetsWhereAccess() throws Exception { SearchRequest query = new SearchRequest(); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java index 5b52698a2..0639dc902 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java @@ -110,7 +110,7 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") @SuppressWarnings("unchecked") public void searchWithRange() @@ -160,7 +160,7 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER }, executionType = ExecutionType.REGRESSION, description = "Check date facet intervals search api") @SuppressWarnings("unchecked") public void searchWithRangeHardend() @@ -258,7 +258,7 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") @SuppressWarnings("unchecked") public void searchWithRangeAndIncludeUpperBound() diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java index ce389980d..4c88a08af 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java @@ -103,7 +103,7 @@ public class FacetedSearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail(section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Checks facet queries for the Search api") + @TestRail(section = { TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER }, executionType = ExecutionType.REGRESSION, description = "Checks facet queries for the Search api") public void searchWithQueryFaceting() throws Exception { SearchRequest query = new SearchRequest(); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java index c99a89ae4..fd266608f 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java @@ -120,7 +120,7 @@ public class PivotFacetedSearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}, executionType = ExecutionType.REGRESSION, description = "Checks nested pivot using Search api") public void searchWithNestedPivoting() diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java index 987f141d1..ff89543fb 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java @@ -155,7 +155,7 @@ public class SearchAFTSInFieldTest extends AbstractE2EFunctionalTest }); } - @Test(priority = 4) + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) public void testSearchInFieldSITE() { String query = "SITE:" + testSite.getId(); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java index 1016869cc..93c67af45 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java @@ -29,6 +29,7 @@ import org.alfresco.rest.search.RestRequestQueryModel; import org.alfresco.rest.search.RestResultBucketsModel; import org.alfresco.rest.search.SearchRequest; import org.alfresco.rest.search.SearchResponse; +import org.alfresco.search.TestGroup; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -92,7 +93,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest *}} * */ - @Test + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) public void searchLevel0() { SearchRequest searchQuery = searchRequestWithAPATHFacet("name:*", "0"); @@ -108,7 +109,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest * Test to test that the facet buckets are returned correctly for sub-level 1/ * Test to search for a searchString, that's unique to the test run and hence stable for any environment */ - @Test + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) public void searchLevel0andIncludeSubLevel1() { SearchRequest searchQuery = searchRequestWithAPATHFacet("name:" + unique_searchString, "1/"); @@ -120,7 +121,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest getFirstBucket(response).assertThat().field("label").contains("1/"); } - @Test + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) public void searchLevel2() { String queryString = "name:cars"; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java index b382bc6f3..58f3e4fbd 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java @@ -9,6 +9,7 @@ package org.alfresco.test.search.functional.searchServices.search; import org.alfresco.rest.model.RestNodeAssociationModelCollection; import org.alfresco.rest.model.RestNodeChildAssociationModel; +import org.alfresco.search.TestGroup; import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; import org.alfresco.utility.data.CustomObjectTypeProperties; import org.alfresco.utility.model.FileModel; @@ -52,7 +53,7 @@ public class SearchSecondaryAssociationTest extends AbstractE2EFunctionalTest waitForMetadataIndexing(file1.getName(), true); } - @Test(priority = 1) + @Test(priority = 1, groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) public void testSearchPathForSecondaryAssociation() throws Exception { String queryPathFolder1 = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java index aa49cf8e5..da43c03f3 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java @@ -102,7 +102,7 @@ public class SearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER }, executionType = ExecutionType.REGRESSION, description = "Tests a search request containing a sort clause.") public void searchWithOneSortClause() { @@ -149,7 +149,7 @@ public class SearchTest extends AbstractSearchServicesE2ETest * processed and the returned order is determined by the second clause. */ @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}, executionType = ExecutionType.REGRESSION, description = "Tests a search request containing a sort clause.") public void searchWithTwoSortClauses() { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java index 5de017a74..1c8bfb600 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java @@ -204,7 +204,7 @@ public class StatsSearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}, executionType = ExecutionType.REGRESSION, description = "Checks errors with stats with Pivot using Search api") public void searchWithStatsAndMutlilevelPivot() diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java index 75b601800..abd65e6bb 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java @@ -76,7 +76,7 @@ public class CascadingIntegrationTest extends AbstractE2EFunctionalTest "failing while removing sharding model"); } - @Test(priority = 1) + @Test(priority = 1, groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) public void testChildPathWhenParentRenamed() throws Exception { // Create Parent folder @@ -119,7 +119,7 @@ public class CascadingIntegrationTest extends AbstractE2EFunctionalTest Assert.assertEquals(descendantCountOfOriginalName, 0, "Old path still has descendants: " + parentQuery); } - @Test(priority = 2) + @Test(priority = 2, groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) public void testGrandChildPathWhenGrandParentRenamed() throws Exception { // Create grand parent folder From 5ae86d2ef842c60218f1ca5aefdeb953ac8035c6 Mon Sep 17 00:00:00 2001 From: Meenal Bhave Date: Wed, 29 Apr 2020 15:12:58 +0100 Subject: [PATCH 48/54] Search-2203: Amended the test to work on non empty shards and to check the appropriate rangeCheck result --- .../solr/admin/SolrE2eAdminTest.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java index 52cbc65f9..b0ea83733 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java @@ -475,17 +475,24 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest checkResponseStatusOk(response); - Integer expand = response.getResponse().body().jsonPath().get("expand"); + Integer expand = response.getResponse().body().jsonPath().get("expand"); // RangeCheck action only applies to DB_ID_RANGE Sharding method, so expect error in other sharding methods and success for DB_ID_RANGE if (ShardingMethod.DB_ID_RANGE.toString().equalsIgnoreCase(getShardMethod())) { // This assertion replicates: testRangeCheckSharding, priority = 21, hence deleting that test as duplicate - Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is not successful when not using Shard DB_ID_RANGE method,"); + // Value -1 is expected when the next shard already has nodes indexed. + Assert.assertTrue((expand == Integer.valueOf(-1) || expand == Integer.valueOf(0)), "RangeCheck should not have been allowed when not using Shard DB_ID_RANGE method,"); + + Integer minDbid = response.getResponse().body().jsonPath().get("minDbid"); + Assert.assertTrue(minDbid > 0, "RangeCheck is not successful when not using Shard DB_ID_RANGE method,"); } else { - Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion should not have been allowed when not using Shard DB_ID_RANGE method,"); + Assert.assertEquals(expand, Integer.valueOf(-1), "RangeCheck should not have been allowed when not using Shard DB_ID_RANGE method,"); + + String exception = response.getResponse().body().jsonPath().get("exception"); + Assert.assertEquals(exception, "ERROR: Wrong document router type:DBIDRouter", "Expansion should not have been allowed when not using Shard DB_ID_RANGE method,"); } } @@ -509,7 +516,7 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest // Expand action only applies to DB_ID_RANGE Sharding method, so expect error in other sharding methods and success for DB_ID_RANGE if (ShardingMethod.DB_ID_RANGE.toString().equalsIgnoreCase(getShardMethod())) { - // This assertion replicates: testExpandSharding, priority = 23, hence deleting that test as duplicate + // This assertion replicates: testExpandSharding, priority = 23, hence deleting that test as duplicate Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is not successful when not using Shard DB_ID_RANGE method,"); } else From ce801db7737852de75561efd8dffd809bcb179a1 Mon Sep 17 00:00:00 2001 From: Keerat Date: Wed, 29 Apr 2020 15:44:12 +0100 Subject: [PATCH 49/54] SEARCH-2121 Adding test group to the @test annotation and not in the test rail --- .../searchServices/search/FacetRangeSearchTest.java | 12 ++++++------ .../searchServices/search/FacetedSearchTest.java | 4 ++-- .../search/PivotFacetedSearchTest.java | 4 ++-- .../functional/searchServices/search/SearchTest.java | 8 ++++---- .../searchServices/search/StatsSearchTest.java | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java index 0639dc902..a44644c43 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java @@ -109,8 +109,8 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "gap")); } - @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}, executionType = ExecutionType.REGRESSION, + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") @SuppressWarnings("unchecked") public void searchWithRange() @@ -159,8 +159,8 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest assertEquals(info.get("endInclusive"), "true"); } - @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER }, executionType = ExecutionType.REGRESSION, + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Check date facet intervals search api") @SuppressWarnings("unchecked") public void searchWithRangeHardend() @@ -257,8 +257,8 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest query.setRanges(ranges); } - @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}, executionType = ExecutionType.REGRESSION, + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") @SuppressWarnings("unchecked") public void searchWithRangeAndIncludeUpperBound() diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java index 4c88a08af..650be58a9 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java @@ -102,8 +102,8 @@ public class FacetedSearchTest extends AbstractSearchServicesE2ETest waitForContentIndexing(file4.getContent(), true); } - @Test - @TestRail(section = { TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER }, executionType = ExecutionType.REGRESSION, description = "Checks facet queries for the Search api") + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @TestRail(section = { TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Checks facet queries for the Search api") public void searchWithQueryFaceting() throws Exception { SearchRequest query = new SearchRequest(); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java index fd266608f..21963b825 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java @@ -119,8 +119,8 @@ public class PivotFacetedSearchTest extends AbstractSearchServicesE2ETest assertPivotResponse(response, "creator", null); } - @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}, + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Checks nested pivot using Search api") public void searchWithNestedPivoting() diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java index da43c03f3..f1f8935d5 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java @@ -101,8 +101,8 @@ public class SearchTest extends AbstractSearchServicesE2ETest response.getContext().assertThat().field("request").isNotEmpty(); } - @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER }, executionType = ExecutionType.REGRESSION, + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Tests a search request containing a sort clause.") public void searchWithOneSortClause() { @@ -148,8 +148,8 @@ public class SearchTest extends AbstractSearchServicesE2ETest * The first clause has always the same value for all matches so the test makes sure the request is correctly * processed and the returned order is determined by the second clause. */ - @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}, executionType = ExecutionType.REGRESSION, + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Tests a search request containing a sort clause.") public void searchWithTwoSortClauses() { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java index 1c8bfb600..7ba363b80 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java @@ -203,8 +203,8 @@ public class StatsSearchTest extends AbstractSearchServicesE2ETest assertTrue((Integer)metricCount.get("countValues") > count, "With the exclude filter there will be more documents than returned"); } - @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH, TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}, + @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Checks errors with stats with Pivot using Search api") public void searchWithStatsAndMutlilevelPivot() From a05d771f6002073037a57e8a0cdf5c2cb783f3c1 Mon Sep 17 00:00:00 2001 From: Keerat Date: Wed, 29 Apr 2020 16:21:41 +0100 Subject: [PATCH 50/54] SEARCH-2121 splitting CMIS tests to run tests in correct config --- .../cmis/SolrSearchByAspectTests.java | 14 +++++++++- .../cmis/SolrSearchByIdTests.java | 17 +++++++++++- .../testdata/search-by-aspect-in-tree.xml | 26 +++++++++++++++++++ .../resources/testdata/search-by-aspect.xml | 22 ---------------- .../testdata/search-by-id-in-tree.xml | 7 +++++ .../test/resources/testdata/search-by-id.xml | 1 - 6 files changed, 62 insertions(+), 25 deletions(-) create mode 100644 e2e-test/src/test/resources/testdata/search-by-aspect-in-tree.xml create mode 100644 e2e-test/src/test/resources/testdata/search-by-id-in-tree.xml diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java index 5c073dd76..91ab34efb 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java @@ -129,7 +129,7 @@ public class SolrSearchByAspectTests extends AbstractCmisE2ETest Utility.waitToLoopTime(getSolrWaitTimeInSeconds()); } - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") @XMLDataConfig(file = "src/test/resources/testdata/search-by-aspect.xml") public void executeSearchByAspect(QueryModel query) throws Exception { @@ -140,4 +140,16 @@ public class SolrSearchByAspectTests extends AbstractCmisE2ETest cmisApi.authenticateUser(testUser); Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); } + + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @XMLDataConfig(file = "src/test/resources/testdata/search-by-aspect-in-tree.xml") + public void executeSearchByAspectInTree(QueryModel query) throws Exception + { + String currentQuery = query.getValue() + .replace("NODE_REF[f1]", tasFolder1.getNodeRef()) + .replace("NODE_REF[s1]", siteDoclibNodeRef); + + cmisApi.authenticateUser(testUser); + Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); + } } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java index 1db07b09b..bb5299211 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java @@ -87,7 +87,7 @@ public class SolrSearchByIdTests extends AbstractCmisE2ETest Utility.waitToLoopTime(getSolrWaitTimeInSeconds()); } - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") @XMLDataConfig(file = "src/test/resources/testdata/search-by-id.xml") public void executeSearchById(QueryModel query) throws Exception { @@ -101,4 +101,19 @@ public class SolrSearchByIdTests extends AbstractCmisE2ETest cmisApi.authenticateUser(testUser); Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); } + + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @XMLDataConfig(file = "src/test/resources/testdata/search-by-id-in-tree.xml") + public void executeSearchByIdInTree(QueryModel query) throws Exception + { + String currentQuery = query.getValue() + .replace("NODE_REF[siteId]", siteDoclibNodeRef) + .replace("NODE_REF[d1]", tasSubFile1.getNodeRefWithoutVersion()) + .replace("NODE_REF[d2]", tasSubFile2.getNodeRefWithoutVersion()) + .replace("NODE_REF[f1]", tasFolder1.getNodeRef()) + .replace("NODE_REF[f1-1]", tasSubFolder1.getNodeRef()); + + cmisApi.authenticateUser(testUser); + Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); + } } diff --git a/e2e-test/src/test/resources/testdata/search-by-aspect-in-tree.xml b/e2e-test/src/test/resources/testdata/search-by-aspect-in-tree.xml new file mode 100644 index 000000000..2f718ccaf --- /dev/null +++ b/e2e-test/src/test/resources/testdata/search-by-aspect-in-tree.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/e2e-test/src/test/resources/testdata/search-by-aspect.xml b/e2e-test/src/test/resources/testdata/search-by-aspect.xml index ef7090d59..cd47c7a92 100644 --- a/e2e-test/src/test/resources/testdata/search-by-aspect.xml +++ b/e2e-test/src/test/resources/testdata/search-by-aspect.xml @@ -1,6 +1,5 @@ - @@ -58,27 +57,6 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/e2e-test/src/test/resources/testdata/search-by-id-in-tree.xml b/e2e-test/src/test/resources/testdata/search-by-id-in-tree.xml new file mode 100644 index 000000000..b3d17d088 --- /dev/null +++ b/e2e-test/src/test/resources/testdata/search-by-id-in-tree.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/e2e-test/src/test/resources/testdata/search-by-id.xml b/e2e-test/src/test/resources/testdata/search-by-id.xml index 9c6cf645e..a34c8463f 100644 --- a/e2e-test/src/test/resources/testdata/search-by-id.xml +++ b/e2e-test/src/test/resources/testdata/search-by-id.xml @@ -6,7 +6,6 @@ - From 0f47d64b29906e275d57c13ededffd77bc333bd4 Mon Sep 17 00:00:00 2001 From: Meenal Bhave Date: Thu, 30 Apr 2020 09:18:49 +0100 Subject: [PATCH 51/54] Merge branch 'master' of https://git.alfresco.com/search_discovery/insightengine into fix/Search-2065 --- .../search/FingerPrintTest.java | 208 +++++++++++------- 1 file changed, 132 insertions(+), 76 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java index 858e8b2f5..ac07b7718 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java @@ -18,9 +18,11 @@ */ package org.alfresco.test.search.functional.searchServices.search; +import static jersey.repackaged.com.google.common.collect.Sets.newHashSet; import static org.testng.Assert.assertTrue; -import org.alfresco.rest.search.SearchNodeModel; +import java.util.Set; + import org.alfresco.rest.search.SearchResponse; import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; import org.alfresco.utility.model.FileModel; @@ -35,12 +37,18 @@ import org.testng.annotations.Test; * Search end point Public API test with finger print. * * @author Michael Suzuki + * @author Meenal Bhave */ public class FingerPrintTest extends AbstractE2EFunctionalTest { - private FileModel file1; - private FileModel file2; private FolderModel folder; + private FileModel fileBanana, fileTaco, fileCat, fileDog, fileOriginal; + /** + * File to be created in dataprep with same content as fileOriginal + * File to be updated in test searchAfterVersionUpdate + * File to be reverted in test searchAfterVersionRevert + */ + private FileModel fileToBeUpdated; @BeforeClass(alwaysRun = true) public void dataPreparation() throws Exception @@ -51,30 +59,39 @@ public class FingerPrintTest extends AbstractE2EFunctionalTest * |-- pangram-banana.txt * |-- pangram-taco.txt * |-- pangram-cat.txt - * |-- dog.txt + * |-- pangram-dog.txt + * |-- original.txt + * |-- toBeUpdated.txt */ folder = new FolderModel("The quick brown fox jumps over"); dataContent.usingUser(testUser).usingSite(testSite).createFolder(folder); - file1 = new FileModel("pangram-banana.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy banana"); - file2 = new FileModel("pangram-taco.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy dog that ate the taco"); + fileBanana = new FileModel("pangram-banana.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy banana"); + fileTaco = new FileModel("pangram-taco.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy dog that ate the taco"); - FileModel file3 = new FileModel("pangram-cat.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy cat"); - FileModel file4 = new FileModel("dog.txt", FileType.TEXT_PLAIN, "The quick brown fox ate the lazy dog"); + fileCat = new FileModel("pangram-cat.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy cat"); + fileDog = new FileModel("pangram-dog.txt", FileType.TEXT_PLAIN, "The quick brown fox ate the lazy dog"); + + // fileOriginal with different content + fileOriginal = new FileModel("original.txt", FileType.TEXT_PLAIN, "This is an original file with some content"); + + // Index a new fileToBeUpdated with content similar to fileOriginal + fileToBeUpdated = new FileModel("toBeUpdated.txt", FileType.TEXT_PLAIN, "This is an original file with some content"); - dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file1); - dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file2); - dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file3); - dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file4); - - waitForContentIndexing(file4.getContent(), true); + dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileBanana); + dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileTaco); + dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileCat); + dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileDog); + + dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileOriginal); + dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileToBeUpdated); // Additional wait implemented to remove inconsistent failures. Ref: Search-1438 for details - waitForIndexing("FINGERPRINT:" + file2.getNodeRefWithoutVersion(), true); - waitForIndexing("FINGERPRINT:" + file3.getNodeRefWithoutVersion(), true); + waitForIndexing("FINGERPRINT:" + fileDog.getNodeRefWithoutVersion(), true); + waitForIndexing("FINGERPRINT:" + fileToBeUpdated.getNodeRefWithoutVersion(), true); } - @Test + @Test(priority = 1) @Bug(id = "MNT-20449") public void makeSureFingerprintQueryWorksAfterMetadataUpdate() throws Exception { @@ -105,86 +122,125 @@ public class FingerPrintTest extends AbstractE2EFunctionalTest * to the files loaded as part of this test. * Note that for fingerprint to work it need a 5 word sequence. */ - @Test + @Test(priority = 2) public void search() { - String uuid = file1.getNodeRefWithoutVersion(); + String uuid = fileBanana.getNodeRefWithoutVersion(); Assert.assertNotNull(uuid); - String fingerprint = String.format("FINGERPRINT:%s", uuid); - SearchResponse response = query(fingerprint); + + String fingerprintQuery = String.format("FINGERPRINT:%s", uuid); + SearchResponse response = queryAsUser(testUser, fingerprintQuery); + int count = response.getEntries().size(); assertTrue(count > 1); - for (SearchNodeModel m : response.getEntries()) - { - String match = m.getModel().getName(); - switch (match) - { - case "pangram.txt": - case "pangram-banana.txt": - case "pangram-taco.txt": - case "pangram-cat.txt": - break; - default: - throw new AssertionError("Not a match to an expected file: " + m.getModel().getName()); - } - m.getModel().assertThat().field("name").isNot("dog.txt"); - m.getModel().assertThat().field("name").isNot("cars.txt"); - } + + Set expectedNames = newHashSet(); + expectedNames.add(fileBanana.getName()); + expectedNames.add(fileTaco.getName()); + expectedNames.add(fileCat.getName()); + + testSearchQueryUnordered(fingerprintQuery, expectedNames, SearchLanguage.AFTS); } - @Test + @Test(priority = 3) public void searchSimilar() { - String uuid = file2.getNodeRefWithoutVersion(); + String uuid = fileTaco.getNodeRefWithoutVersion(); Assert.assertNotNull(uuid); // In the response entity there is a score of each doc, change below threshold to bring more like or less. - String fingerprint = String.format("FINGERPRINT:%s_68", uuid); - SearchResponse response = query(fingerprint); + String fingerprintQuery = String.format("FINGERPRINT:%s_68", uuid); + SearchResponse response = query(fingerprintQuery); int count = response.getEntries().size(); assertTrue(count > 1); - - for (SearchNodeModel m : response.getEntries()) - { - switch (m.getModel().getName()) - { - case "pangram.txt": - case "pangram-taco.txt": - break; - default: - throw new AssertionError("Not a match to an expected file: " + m.getModel().getName()); - } - m.getModel().assertThat().field("name").isNot("pangram-banana.txt"); - m.getModel().assertThat().field("name").isNot("pangram-cat.txt"); - m.getModel().assertThat().field("name").isNot("dog.txt"); - m.getModel().assertThat().field("name").isNot("cars.txt"); - } + + Set expectedNames = newHashSet(); + expectedNames.add(fileTaco.getName()); + + testSearchQueryUnordered(fingerprintQuery, expectedNames, SearchLanguage.AFTS); } - @Test + @Test(priority = 4) public void searchSimilar67Percent() { - String uuid = file2.getNodeRefWithoutVersion(); + String uuid = fileTaco.getNodeRefWithoutVersion(); Assert.assertNotNull(uuid); - String fingerprint = String.format("FINGERPRINT:%s_68", uuid); - SearchResponse response = query(fingerprint); + + // Check that: When asked for less overlap of 67%: More documents are matched + String fingerprintQuery = String.format("FINGERPRINT:%s_67", uuid); + SearchResponse response = query(fingerprintQuery); + int count = response.getEntries().size(); assertTrue(count > 1); - for (SearchNodeModel m : response.getEntries()) - { - switch (m.getModel().getName()) - { - case "pangram.txt": - case "pangram-taco.txt": - case "pangram-cat.txt": - break; - default: - throw new AssertionError("Not a match to an expected file: " + m.getModel().getName()); - } - m.getModel().assertThat().field("name").isNot("pangram-banana.txt"); - m.getModel().assertThat().field("name").isNot("dog.txt"); - m.getModel().assertThat().field("name").isNot("cars.txt"); - } + + Set expectedNames = newHashSet(); + expectedNames.add(fileTaco.getName()); + expectedNames.add(fileBanana.getName()); + expectedNames.add(fileCat.getName()); + + testSearchQueryUnordered(fingerprintQuery, expectedNames, SearchLanguage.AFTS); + } + + @Test(priority = 5, enabled = false) + @Bug(id = "SEARCH-2065") + public void searchAfterVersionUpdate() + { + // Check that fileToBeUpdated is found with a fingerprint query with fileOriginal: as they have similar content + boolean found = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true); + Assert.assertTrue(found, "Matching File Not found in results for Fingerprint Query with original file"); + + // Update content of the fileToBeUpdated to match file1 in the dataprep + String newFileContent = "The quick brown fox jumps over the updated file"; + fileToBeUpdated.setContent(newFileContent); + dataContent.usingUser(testUser).usingSite(testSite).usingResource(fileToBeUpdated).updateContent(newFileContent); + + // Wair for the new version of the file to be indexed + assertTrue(waitForContentIndexing(fileToBeUpdated.getContent(), true)); + + // Check that fileToBeUpdated is NOT found with a fingerprint query with fileOriginal + boolean notFound = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), false); + Assert.assertTrue(notFound, "Updated File unexpectedly found in results for Fingerprint Query with original file"); + + // Check that fileToBeUpdated is found with a fingerprint query with file1 + found = isContentInSearchResults("FINGERPRINT:" + fileBanana.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true); + Assert.assertTrue(found, "Update File Not found in results for Fingerprint Query with updated content"); + } + + @Test(priority = 6, enabled = false) + @Bug(id = "SEARCH-2065") + public void searchAfterVersionRevert() throws Exception + { + // Revert fileToBeUpdated to previous version + restClient.authenticateUser(testUser).withCoreAPI().usingNode(fileToBeUpdated).revertVersion("1.0", "{}"); + String revertedContent = restClient.authenticateUser(testUser).withCoreAPI().usingNode(fileToBeUpdated).getVersionContent("1.2").toString(); + Assert.assertEquals(revertedContent, fileOriginal.getContent(), "Reverted content does not match Original"); + + // Wair for the new version of the file to be indexed + assertTrue(waitForContentIndexing(fileToBeUpdated.getContent(), true)); + + // Check that fileToBeUpdated is found with a fingerprint query with fileOriginal + boolean found = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true); + Assert.assertTrue(found, "File not found in results for Fingerprint Query with original file after reverting version changes"); + + // Check that fileToBeUpdated is NOT found with a fingerprint query with file1 + boolean notFound = isContentInSearchResults("FINGERPRINT:" + fileBanana.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), false); + Assert.assertTrue(notFound, "File appears in the results for Fingerprint Query even after reverting content changes"); + } + + @Test(priority = 7, enabled = false) + @Bug(id = "SEARCH-2065") + public void searchAfterVersionDelete() throws Exception + { + // Revert fileToBeUpdated to previous version + restClient.authenticateUser(testUser).withCoreAPI().usingNode(fileToBeUpdated).deleteNodeVersion("1.2"); + + // Check that fileToBeUpdated is NOT found with a fingerprint query with fileOriginal + boolean notFound = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), false); + Assert.assertTrue(notFound, "Updated File unexpectedly found in results for Fingerprint Query with original file"); + + // Check that fileToBeUpdated is found with a fingerprint query with file1 + boolean found = isContentInSearchResults("FINGERPRINT:" + fileBanana.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true); + Assert.assertTrue(found, "Update File Not found in results for Fingerprint Query with updated content"); } } From ccdecbdeb28f3ab0fa20c6efda7aa4497e8c320a Mon Sep 17 00:00:00 2001 From: mbhave Date: Thu, 30 Apr 2020 10:27:06 +0100 Subject: [PATCH 52/54] Search-2203: Missed out hunk for testExpand from the commit --- .../searchServices/solr/admin/SolrE2eAdminTest.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java index b0ea83733..20666ad8c 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java @@ -517,11 +517,17 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest if (ShardingMethod.DB_ID_RANGE.toString().equalsIgnoreCase(getShardMethod())) { // This assertion replicates: testExpandSharding, priority = 23, hence deleting that test as duplicate - Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is not successful when not using Shard DB_ID_RANGE method,"); + Assert.assertTrue((expand == Integer.valueOf(-1) || expand == Integer.valueOf(0)), "Expansion is not successful when not using Shard DB_ID_RANGE method,"); + if (expand == Integer.valueOf(-1)) + { + String exceptionExpected = "Expansion cannot occur if max DBID in the index is more then 75% of range."; + String exception = response.getResponse().body().jsonPath().get("exception"); + Assert.assertEquals(exception, exceptionExpected, "Expansion failed with unexpected Exception while using DB_ID_RANGE sharding"); + } } else { - Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion should not have been allowed when not using Shard DB_ID_RANGE method,"); + Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion should not have been allowed when not using Shard DB_ID_RANGE method"); } } From 0875a04a9caf771aa3a44ed6318d8f32b23a106d Mon Sep 17 00:00:00 2001 From: Elia Porciani Date: Fri, 1 May 2020 08:45:02 +0100 Subject: [PATCH 53/54] SEARCH-2129: Static semaphore for running and writing Tracker Threads, that avoids parallel execution of the same job. Standarization for logger naming and declaration. --- .../org/alfresco/solr/InformationServer.java | 2 +- .../alfresco/solr/SolrInformationServer.java | 8 +- .../solr/tracker/AbstractTracker.java | 686 +++++++++--------- ...orkerRunnable.java => AbstractWorker.java} | 124 ++-- .../org/alfresco/solr/tracker/AclTracker.java | 307 ++++---- .../alfresco/solr/tracker/CascadeTracker.java | 138 ++-- .../alfresco/solr/tracker/CommitTracker.java | 28 +- .../alfresco/solr/tracker/ContentTracker.java | 107 +-- .../solr/tracker/MetadataTracker.java | 651 ++++++++++------- .../alfresco/solr/tracker/ModelTracker.java | 19 + .../alfresco/solr/tracker/QueueHandler.java | 24 - .../solr/tracker/SlaveCoreStatePublisher.java | 21 + .../alfresco/solr/tracker/ThreadHandler.java | 104 --- .../org/alfresco/solr/tracker/TrackerJob.java | 5 + .../pool/DefaultTrackerPoolFactory.java | 2 +- .../templates/rerank/conf/solrcore.properties | 12 +- .../solr/AbstractAlfrescoDistributedIT.java | 3 +- .../org/alfresco/solr/SolrITInitializer.java | 3 +- .../solr/basics/SolrResponsesComparator.java | 61 +- .../solr/tracker/ContentTrackerIT.java | 10 +- ...dExpandDbidRangeAlfrescoSolrTrackerIT.java | 13 + .../pool/DefaultTrackerPoolFactoryTest.java | 14 +- .../transformer/CachedDocTransformerIT.java | 16 +- .../alfresco/solr/client/AclChangeSets.java | 145 ++-- .../solr/client/SOLRAPIQueueClient.java | 2 +- .../alfresco/solr/client/Transactions.java | 15 +- 26 files changed, 1349 insertions(+), 1171 deletions(-) rename search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/{AbstractWorkerRunnable.java => AbstractWorker.java} (54%) delete mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/QueueHandler.java delete mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ThreadHandler.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java index e1b24d82b..7588b2af2 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java @@ -148,7 +148,7 @@ public interface InformationServer extends InformationServerCollectionProvider IndexHealthReport reportIndexTransactions(Long minTxId, IOpenBitSet txIdsInDb, long maxTxId) throws IOException; - List getDocsWithUncleanContent(int start, int rows) throws IOException; + List getDocsWithUncleanContent() throws IOException; void updateContentToIndexAndCache(long dbId, String tenant) throws Exception; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index ba05c9e41..7f23fe75b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -663,7 +663,7 @@ public class SolrInformationServer implements InformationServer } @Override - public List getDocsWithUncleanContent(int start, int rows) throws IOException + public List getDocsWithUncleanContent() throws IOException { RefCounted refCounted = null; try @@ -688,7 +688,7 @@ public class SolrInformationServer implements InformationServer * in current snapshot of the index. * * The code below runs every two minutes and purges transactions from the - * cleanContentCache that is more then 20 minutes old. + * cleanContentCache that is more than 20 minutes old. * */ long purgeTime = System.currentTimeMillis(); @@ -721,7 +721,11 @@ public class SolrInformationServer implements InformationServer DelegatingCollector delegatingCollector = new TxnCacheFilter(cleanContentCache); //Filter transactions that have already been processed. delegatingCollector.setLastDelegate(collector); searcher.search(dirtyOrNewContentQuery(), delegatingCollector); + + LOGGER.debug("{}-[CORE {}] Processing {} documents with content to be indexed", Thread.currentThread().getId(), core.getName(), collector.getTotalHits()); + + if(collector.getTotalHits() == 0) { return docIds; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java index 038f56664..624f26c91 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java @@ -1,361 +1,327 @@ -/* - * Copyright (C) 2005-2019 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.tracker; - -import static java.util.Optional.ofNullable; - -import java.net.ConnectException; -import java.net.SocketTimeoutException; -import java.util.Properties; -import java.util.concurrent.Semaphore; - -import org.alfresco.service.cmr.repository.StoreRef; -import org.alfresco.solr.IndexTrackingShutdownException; -import org.alfresco.solr.InformationServer; -import org.alfresco.solr.TrackerState; -import org.alfresco.solr.client.SOLRAPIClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Abstract base class that provides common {@link Tracker} behaviour. - * - * @author Matt Ward - */ -public abstract class AbstractTracker implements Tracker -{ - static final long TIME_STEP_32_DAYS_IN_MS = 1000 * 60 * 60 * 24 * 32L; - static final long TIME_STEP_1_HR_IN_MS = 60 * 60 * 1000L; - static final String SHARD_METHOD_DBID = "DB_ID"; - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractTracker.class); - - protected Properties props; - protected SOLRAPIClient client; - InformationServer infoSrv; - protected String coreName; - StoreRef storeRef; - long batchCount; - TrackerStats trackerStats; - boolean runPostModelLoadInit = true; - private int maxLiveSearchers; - private volatile boolean shutdown = false; - - private Semaphore runLock = new Semaphore(1, true); - private Semaphore writeLock = new Semaphore(1, true); - - protected volatile TrackerState state; - protected int shardCount; - protected int shardInstance; - String shardMethod; - protected boolean transformContent; - String shardTemplate; - protected volatile boolean rollback; - /** - * When rollback is set, original error is also gathered in order to provide detailed logging. - */ - protected Throwable rollbackCausedBy; - protected final Type type; - protected final String trackerId; - - /* - * A thread handler can be used by subclasses, but they have to intentionally instantiate it. - */ - ThreadHandler threadHandler; - - /** - * Default constructor, strictly for testing. - */ - protected AbstractTracker(Type type) - { - this.type = type; - this.trackerId = type + "@" + hashCode(); - } - - protected AbstractTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer,Type type) - { - this.props = p; - this.client = client; - this.coreName = coreName; - this.infoSrv = informationServer; - - storeRef = new StoreRef(p.getProperty("alfresco.stores", "workspace://SpacesStore")); - batchCount = Integer.parseInt(p.getProperty("alfresco.batch.count", "5000")); - maxLiveSearchers = Integer.parseInt(p.getProperty("alfresco.maxLiveSearchers", "2")); - - shardCount = Integer.parseInt(p.getProperty("shard.count", "1")); - shardInstance = Integer.parseInt(p.getProperty("shard.instance", "0")); - shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); - - shardTemplate = p.getProperty("alfresco.template", ""); - - transformContent = Boolean.parseBoolean(p.getProperty("alfresco.index.transformContent", "true")); - - this.trackerStats = this.infoSrv.getTrackerStats(); - - this.type = type; - - this.trackerId = type + "@" + hashCode(); - } - - - /** - * Subclasses must implement behaviour that completes the following steps, in order: - * - *
      - *
    1. Purge
      2. - *
    2. Reindex
      3. - *
    3. Index
      4. - *
    4. Track repository
      5. - *
    - * - * @param iterationId an identifier which is uniquely associated with a given iteration. - */ - protected abstract void doTrack(String iterationId) throws Throwable; - - - private boolean assertTrackerStateRemainsNull() { - - /* - * This assertion is added to accommodate DistributedAlfrescoSolrTrackerRaceTest. - * The sleep is needed to allow the test case to add a txn into the queue before - * the tracker makes its call to pull transactions from the test repo client. - */ - - try - { - Thread.sleep(5000); - } - catch(Exception e) - { - // Ignore - } - - - /* - * This ensures that getTrackerState does not have the side effect of setting the - * state instance variable. This allows classes outside of the tracker framework - * to safely call getTrackerState without interfering with the trackers design. - */ - - getTrackerState(); - - return state == null; - - } - /** - * Template method - subclasses must implement the {@link Tracker}-specific indexing - * by implementing the abstract method {@link #doTrack(String)}. - */ - @Override - public void track() - { - String iterationId = "IT #" + System.currentTimeMillis(); - - if(runLock.availablePermits() == 0) - { - LOGGER.info("[{} / {} / {}] Tracker already registered.", coreName, trackerId, iterationId); - return; - } - - try - { - /* - * The runLock ensures that for each tracker type (metadata, content, commit, cascade) only one tracker will - * be running at a time. - */ - runLock.acquire(); - - if (state==null && Boolean.parseBoolean(System.getProperty("alfresco.test", "false"))) - { - assert(assertTrackerStateRemainsNull()); - } - - if(this.state == null) - { - this.state = getTrackerState(); - - LOGGER.debug("[{} / {} / {}] Global Tracker State set to: {}", coreName, trackerId, iterationId, this.state.toString()); - this.state.setRunning(true); - } - else - { - continueState(); - this.state.setRunning(true); - } - - infoSrv.registerTrackerThread(); - - try - { - doTrack(iterationId); - } - catch(IndexTrackingShutdownException t) - { - setRollback(true, t); - LOGGER.info("[{} / {} / {}] Tracking cycle stopped. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); - } - catch(Throwable t) - { - setRollback(true, t); - if (t instanceof SocketTimeoutException || t instanceof ConnectException) - { - LOGGER.warn("[{} / {} / {}] Tracking communication timed out. See the stacktrace below for further details.", coreName, trackerId, iterationId); - LOGGER.debug("[{} / {} / {}] Stack trace", coreName, trackerId, iterationId, t); - } - else - { - LOGGER.error("[{} / {} / {}] Tracking failure. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); - } - } - } - catch (InterruptedException e) - { - LOGGER.error("[{} / {} / {}] Semaphore interruption. See the stacktrace below for further details.", coreName, trackerId, iterationId, e); - } - finally - { - infoSrv.unregisterTrackerThread(); - ofNullable(state).ifPresent(tstate -> { - // During a rollback state is set to null. - state.setRunning(false); - state.setCheck(false); - }); - runLock.release(); - } - } - - public boolean getRollback() - { - return this.rollback; - } - - public Throwable getRollbackCausedBy() - { - return this.rollbackCausedBy; - } - - public void setRollback(boolean rollback, Throwable rollbackCausedBy) - { - this.rollback = rollback; - this.rollbackCausedBy = rollbackCausedBy; - } - - private void continueState() - { - infoSrv.continueState(state); - state.incrementTrackerCycles(); - } - - public synchronized void invalidateState() - { - state = null; - } - - @Override - public synchronized TrackerState getTrackerState() - { - if(this.state != null) - { - return this.state; - } - else - { - return this.infoSrv.getTrackerInitialState(); - } - } - - /** - * Allows time for the scheduled asynchronous tasks to complete - */ - synchronized void waitForAsynchronous() - { - AbstractWorkerRunnable currentRunnable = this.threadHandler.peekHeadReindexWorker(); - while (currentRunnable != null) - { - checkShutdown(); - synchronized (this) - { - try - { - wait(100); - } - catch (InterruptedException e) - { - // Nothing to be done here - } - } - currentRunnable = this.threadHandler.peekHeadReindexWorker(); - } - } - - int getMaxLiveSearchers() - { - return maxLiveSearchers; - } - - void checkShutdown() - { - if(shutdown) - { - throw new IndexTrackingShutdownException(); - } - } - - @Override - public boolean isAlreadyInShutDownMode() - { - return shutdown; - } - - @Override - public void setShutdown(boolean shutdown) - { - this.shutdown = shutdown; - } - - @Override - public void shutdown() - { - setShutdown(true); - if(this.threadHandler != null) - { - threadHandler.shutDownThreadPool(); - } - } - - public Semaphore getWriteLock() - { - return this.writeLock; - } - - Semaphore getRunLock() - { - return this.runLock; - } - - public Properties getProps() - { - return props; - } - - public Type getType() - { - return type; - } +/* + * Copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.tracker; + +import static java.util.Optional.ofNullable; + +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.util.Properties; +import java.util.concurrent.Semaphore; + +import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.solr.IndexTrackingShutdownException; +import org.alfresco.solr.InformationServer; +import org.alfresco.solr.TrackerState; +import org.alfresco.solr.client.SOLRAPIClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract base class that provides common {@link Tracker} behaviour. + * + * @author Matt Ward + */ +public abstract class AbstractTracker implements Tracker +{ + static final long TIME_STEP_32_DAYS_IN_MS = 1000 * 60 * 60 * 24 * 32L; + static final long TIME_STEP_1_HR_IN_MS = 60 * 60 * 1000L; + static final String SHARD_METHOD_DBID = "DB_ID"; + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractTracker.class); + + protected Properties props; + protected SOLRAPIClient client; + InformationServer infoSrv; + protected String coreName; + StoreRef storeRef; + long batchCount; + TrackerStats trackerStats; + boolean runPostModelLoadInit = true; + private int maxLiveSearchers; + private volatile boolean shutdown = false; + + protected volatile TrackerState state; + protected int shardCount; + protected int shardInstance; + String shardMethod; + protected boolean transformContent; + String shardTemplate; + protected volatile boolean rollback; + /** + * When rollback is set, original error is also gathered in order to provide detailed logging. + */ + protected Throwable rollbackCausedBy; + protected final Type type; + protected final String trackerId; + + /** + * Default constructor, strictly for testing. + */ + protected AbstractTracker(Type type) + { + this.type = type; + this.trackerId = type + "@" + hashCode(); + } + + protected AbstractTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer,Type type) + { + this.props = p; + this.client = client; + this.coreName = coreName; + this.infoSrv = informationServer; + + storeRef = new StoreRef(p.getProperty("alfresco.stores", "workspace://SpacesStore")); + batchCount = Integer.parseInt(p.getProperty("alfresco.batch.count", "5000")); + maxLiveSearchers = Integer.parseInt(p.getProperty("alfresco.maxLiveSearchers", "2")); + + shardCount = Integer.parseInt(p.getProperty("shard.count", "1")); + shardInstance = Integer.parseInt(p.getProperty("shard.instance", "0")); + shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); + + shardTemplate = p.getProperty("alfresco.template", ""); + + transformContent = Boolean.parseBoolean(p.getProperty("alfresco.index.transformContent", "true")); + + this.trackerStats = this.infoSrv.getTrackerStats(); + + this.type = type; + + this.trackerId = type + "@" + hashCode(); + } + + + /** + * Subclasses must implement behaviour that completes the following steps, in order: + * + *
      + *
    1. Purge
      2. + *
    2. Reindex
      3. + *
    3. Index
      4. + *
    4. Track repository
      5. + *
    + * + * @param iterationId an identifier which is uniquely associated with a given iteration. + */ + protected abstract void doTrack(String iterationId) throws Throwable; + + + private boolean assertTrackerStateRemainsNull() { + + /* + * This assertion is added to accommodate DistributedAlfrescoSolrTrackerRaceTest. + * The sleep is needed to allow the test case to add a txn into the queue before + * the tracker makes its call to pull transactions from the test repo client. + */ + + try + { + Thread.sleep(5000); + } + catch(Exception e) + { + // Ignore + } + + + /* + * This ensures that getTrackerState does not have the side effect of setting the + * state instance variable. This allows classes outside of the tracker framework + * to safely call getTrackerState without interfering with the trackers design. + */ + + getTrackerState(); + + return state == null; + + } + /** + * Template method - subclasses must implement the {@link Tracker}-specific indexing + * by implementing the abstract method {@link #doTrack(String)}. + */ + @Override + public void track() + { + String iterationId = "IT #" + System.currentTimeMillis(); + + if(getRunLock().availablePermits() == 0) + { + LOGGER.info("[{} / {} / {}] Tracker already registered.", coreName, trackerId, iterationId); + return; + } + + try + { + /* + * The runLock ensures that for each tracker type (metadata, content, commit, cascade) only one tracker will + * be running at a time. + */ + getRunLock().acquire(); + + if (state==null && Boolean.parseBoolean(System.getProperty("alfresco.test", "false"))) + { + assert(assertTrackerStateRemainsNull()); + } + + if(this.state == null) + { + this.state = getTrackerState(); + + LOGGER.debug("[{} / {} / {}] Global Tracker State set to: {}", coreName, trackerId, iterationId, this.state.toString()); + this.state.setRunning(true); + } + else + { + continueState(); + this.state.setRunning(true); + } + + infoSrv.registerTrackerThread(); + + try + { + doTrack(iterationId); + } + catch(IndexTrackingShutdownException t) + { + setRollback(true, t); + LOGGER.info("[{} / {} / {}] Tracking cycle stopped. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); + } + catch(Throwable t) + { + setRollback(true, t); + if (t instanceof SocketTimeoutException || t instanceof ConnectException) + { + LOGGER.warn("[{} / {} / {}] Tracking communication timed out. See the stacktrace below for further details.", coreName, trackerId, iterationId); + LOGGER.debug("[{} / {} / {}] Stack trace", coreName, trackerId, iterationId, t); + } + else + { + LOGGER.error("[{} / {} / {}] Tracking failure. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); + } + } + } + catch (InterruptedException e) + { + LOGGER.error("[{} / {} / {}] Semaphore interruption. See the stacktrace below for further details.", coreName, trackerId, iterationId, e); + } + finally + { + infoSrv.unregisterTrackerThread(); + ofNullable(state).ifPresent(tstate -> { + // During a rollback state is set to null. + state.setRunning(false); + state.setCheck(false); + }); + getRunLock().release(); + } + } + + public boolean getRollback() + { + return this.rollback; + } + + public Throwable getRollbackCausedBy() + { + return this.rollbackCausedBy; + } + + public void setRollback(boolean rollback, Throwable rollbackCausedBy) + { + this.rollback = rollback; + this.rollbackCausedBy = rollbackCausedBy; + } + + private void continueState() + { + infoSrv.continueState(state); + state.incrementTrackerCycles(); + } + + public synchronized void invalidateState() + { + state = null; + } + + @Override + public synchronized TrackerState getTrackerState() + { + if(this.state != null) + { + return this.state; + } + else + { + return this.infoSrv.getTrackerInitialState(); + } + } + + int getMaxLiveSearchers() + { + return maxLiveSearchers; + } + + void checkShutdown() + { + if(shutdown) + { + throw new IndexTrackingShutdownException(); + } + } + + @Override + public boolean isAlreadyInShutDownMode() + { + return shutdown; + } + + @Override + public void setShutdown(boolean shutdown) + { + this.shutdown = shutdown; + } + + @Override + public void shutdown() + { + setShutdown(true); + } + + /** + * Trackers implementing this method should decide if the Write Lock is applied + * globally for every Tracker Thread (static) or locally for each running Thread + */ + public abstract Semaphore getWriteLock(); + + /** + * Trackers implementing this method should decide if the Run Lock is applied + * globally for every Tracker Thread (static) or locally for each running Thread + */ + public abstract Semaphore getRunLock(); + + public Properties getProps() + { + return props; + } + + public Type getType() + { + return type; + } } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorker.java similarity index 54% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorker.java index 2aa335d98..40c126319 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorker.java @@ -1,68 +1,56 @@ -/* - * Copyright (C) 2005-2014 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.tracker; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -abstract class AbstractWorkerRunnable implements Runnable -{ - protected final static Logger log = LoggerFactory.getLogger(AbstractWorkerRunnable.class); - - QueueHandler queueHandler; - - public AbstractWorkerRunnable(QueueHandler qh) - { - this.queueHandler = qh; - } - - /* - * (non-Javadoc) - * @see java.lang.Runnable#run() - */ - @Override - public void run() - { - boolean failed = true; - Exception failCausedBy = null; - try - { - doWork(); - failed = false; - } - catch (Exception e) - { - log.warn("Index tracking batch hit an unrecoverable error ", e); - failCausedBy = e; - } - finally - { - // Triple check that we get the queue state right - queueHandler.removeFromQueueAndProdHead(this); - if(failed) - { - onFail(failCausedBy); - } - } - } - - abstract protected void doWork() throws Exception; - - abstract protected void onFail(Throwable failCausedBy); -} +/* + * Copyright (C) 2005-2020 Alfresco Software Limited. + * This file is part of Alfresco + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.tracker; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Synchronous Tracking Worker. + * + * @author aborroy + * + */ +public abstract class AbstractWorker +{ + protected final static Logger LOGGER = LoggerFactory.getLogger(AbstractWorker.class); + + public void run() + { + boolean failed = true; + Exception failCausedBy = null; + try + { + doWork(); + failed = false; + } + catch (Exception e) + { + LOGGER.warn("Index tracking batch hit an unrecoverable error ", e); + failCausedBy = e; + } + finally + { + if (failed) + { + onFail(failCausedBy); + } + } + } + + abstract protected void doWork() throws Exception; + + abstract protected void onFail(Throwable failCausedBy); +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java index 722e6ba26..5887f16ca 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java @@ -20,13 +20,22 @@ package org.alfresco.solr.tracker; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import com.google.common.collect.Lists; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.AuthenticationException; import org.alfresco.repo.index.shard.ShardMethodEnum; @@ -56,8 +65,11 @@ public class AclTracker extends AbstractTracker { protected final static Logger LOGGER = LoggerFactory.getLogger(AclTracker.class); - private static final int DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE = 100; - private static final int DEFAULT_ACL_BATCH_SIZE = 10; + private static final int DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE = 2000; + private static final int DEFAULT_ACL_BATCH_SIZE = 100; + private static final int DEFAULT_ACL_TRACKER_MAX_PARALLELISM = 32; + + private int aclTrackerParallelism; private int changeSetAclsBatchSize = DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE; private int aclBatchSize = DEFAULT_ACL_BATCH_SIZE; @@ -69,6 +81,23 @@ public class AclTracker extends AbstractTracker private ConcurrentLinkedQueue aclsToIndex = new ConcurrentLinkedQueue(); private ConcurrentLinkedQueue aclsToPurge = new ConcurrentLinkedQueue(); private DocRouter docRouter; + + private ForkJoinPool forkJoinPool; + + // Share run and write locks across all AclTracker threads + private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); + @Override + public Semaphore getWriteLock() + { + return WRITE_LOCK_BY_CORE.get(coreName); + } + @Override + public Semaphore getRunLock() + { + return RUN_LOCK_BY_CORE.get(coreName); + } + /** * Default constructor, for testing. @@ -82,11 +111,18 @@ public class AclTracker extends AbstractTracker String coreName, InformationServer informationServer) { super(p, client, coreName, informationServer, Tracker.Type.ACL); - changeSetAclsBatchSize = Integer.parseInt(p.getProperty("alfresco.changeSetAclsBatchSize", "100")); - aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize", "10")); + changeSetAclsBatchSize = Integer.parseInt(p.getProperty("alfresco.changeSetAclsBatchSize", + String.valueOf(DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE))); + aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize", String.valueOf(DEFAULT_ACL_BATCH_SIZE))); shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod)); - threadHandler = new ThreadHandler(p, coreName, "AclTracker"); + + aclTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.aclTrackerMaxParallelism", + String.valueOf(DEFAULT_ACL_TRACKER_MAX_PARALLELISM))); + forkJoinPool = new ForkJoinPool(aclTrackerParallelism); + + RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } @Override @@ -342,7 +378,7 @@ public class AclTracker extends AbstractTracker AclChangeSet firstChangeSet = firstChangeSets.getAclChangeSets().get(0); long firstChangeSetCommitTime = firstChangeSet.getCommitTimeMs(); state.setLastGoodChangeSetCommitTimeInIndex(firstChangeSetCommitTime); - setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets, state); + setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets.getAclChangeSets(), state); } } @@ -384,7 +420,7 @@ public class AclTracker extends AbstractTracker firstChangeSets = client.getAclChangeSets(null, 0l, null, 2000L, 1); } - setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets, state); + setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets.getAclChangeSets(), state); Long maxChangeSetCommitTimeInRepo = firstChangeSets.getMaxChangeSetCommitTime(); Long maxChangeSetIdInRepo = firstChangeSets.getMaxChangeSetId(); if (maxChangeSetCommitTimeInRepo != null && maxChangeSetIdInRepo != null) @@ -634,6 +670,15 @@ public class AclTracker extends AbstractTracker } /** + * Every ACL Change Set contains a list of ACLs to be indexed. + * This method gets ACL Change Sets from Alfresco Repository to be indexed. + * + * The indexing is performed in batches of ACL Change Sets and the Tracker Status + * is updated in batched of ACLs. + * + * Tracker Status contains the Commit Time from the latest ACL Change Set indexed, + * so new operations can be retrieved from Repository starting with that time. + * * @throws AuthenticationException * @throws IOException * @throws JSONException @@ -646,10 +691,11 @@ public class AclTracker extends AbstractTracker boolean upToDate = false; AclChangeSets aclChangeSets; BoundedDeque changeSetsFound = new BoundedDeque(100); - HashSet changeSetsIndexed = new LinkedHashSet(); long totalAclCount = 0; int aclCount = 0; + LOGGER.info("{}-[CORE {}] Tracking ACLs", Thread.currentThread().getId(), coreName); + do { try @@ -664,12 +710,12 @@ public class AclTracker extends AbstractTracker this.state = getTrackerState(); - Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound, state.getLastGoodChangeSetCommitTimeInIndex()); + Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound, + state.getLastChangeSetCommitTimeOnServer() == 0 ? state.getLastGoodChangeSetCommitTimeInIndex() + : state.getLastChangeSetCommitTimeOnServer()); aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, state.getTimeToStopIndexing()); - setLastChangeSetIdAndCommitTimeInTrackerState(aclChangeSets, state); - if (aclChangeSets.getAclChangeSets().size() > 0) { LOGGER.info("{}-[CORE {}] Found {} ACL change sets after lastTxCommitTime {}, ACL Change Sets from {} to {}", @@ -686,90 +732,64 @@ public class AclTracker extends AbstractTracker Thread.currentThread().getId(), coreName, fromCommitTime); } - ArrayList changeSetBatch = new ArrayList(); - for (int i = 0; i < aclChangeSets.getAclChangeSets().size(); i++) - { - - AclChangeSet changeSet = aclChangeSets.getAclChangeSets().get(i); - - boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() && - infoSrv.aclChangeSetInIndex(changeSet.getId(), true)); - - if (isInIndex) - { - // Logging progress for large ACL Change Set tracking every 100 tracked ACLs - if (LOGGER.isTraceEnabled()) - { - LOGGER.trace("{}-[CORE {}] Tracking {} of {} ACL Change Sets. Change Set Id was already indexed: {}", - Thread.currentThread().getId(), coreName, i + 1, aclChangeSets.getAclChangeSets().size(), changeSet.getId()); - } - changeSetsFound.add(changeSet); - } - else - { - - // Logging progress for ACL Change Set - if (LOGGER.isTraceEnabled()) - { - LOGGER.trace("{}-[CORE {}] Tracking {} of {} ACL Change Sets. Current Change Set Id to be indexed: {}", - Thread.currentThread().getId(), coreName, i + 1, aclChangeSets.getAclChangeSets().size(), changeSet.getId()); - } - - // Make sure we do not go ahead of where we started - we will check the holes here - // correctly next time - if (changeSet.getCommitTimeMs() > state.getTimeToStopIndexing()) { - upToDate = true; - break; - } - - changeSetBatch.add(changeSet); - if (getAclCount(changeSetBatch) > changeSetAclsBatchSize) { - aclCount += indexBatchOfChangeSets(changeSetBatch); - totalAclCount += aclCount; - - for (AclChangeSet scheduled : changeSetBatch) { - changeSetsFound.add(scheduled); - changeSetsIndexed.add(scheduled); + // Ignore indexed ACL Change Sets + aclChangeSets = new AclChangeSets(aclChangeSets.getAclChangeSets().stream() + .filter(changeSet -> { + try + { + boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() && + infoSrv.aclChangeSetInIndex(changeSet.getId(), true)); + if (LOGGER.isTraceEnabled()) + { + LOGGER.trace("{}-[CORE {}] Skipping change Set Id {} as it was already indexed", + Thread.currentThread().getId(), coreName, changeSet.getId()); + } + return !isInIndex; } - changeSetBatch.clear(); - } - } + catch (IOException e) + { + LOGGER.warn( + "{}-[CORE {}] Error catched while checking if ACL Change Set {} was in index", + Thread.currentThread().getId(), coreName, changeSet.getId(), e); + return true; + } + }) + .collect(Collectors.toList())); - if (aclCount > batchCount) { - if (super.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) { - indexAclChangeSetAfterAsynchronous(changeSetsIndexed, state); - long endElapsed = System.nanoTime(); - trackerStats.addElapsedAclTime(aclCount, endElapsed - startElapsed); - startElapsed = endElapsed; - aclCount = 0; - } - } - checkShutdown(); - } - if (!changeSetBatch.isEmpty()) { - if (getAclCount(changeSetBatch) > 0) { - aclCount += indexBatchOfChangeSets(changeSetBatch); - totalAclCount += aclCount; - } - for (AclChangeSet scheduled : changeSetBatch) { - changeSetsFound.add(scheduled); - changeSetsIndexed.add(scheduled); - } - changeSetBatch.clear(); - } - if(changeSetsIndexed.size() > 0) + // Make sure we do not go ahead of where we started - we will check the holes here + // correctly next time + if (aclChangeSets.getAclChangeSets().stream() + .filter(changeSet -> changeSet.getCommitTimeMs() > state.getTimeToStopIndexing()).findAny().isPresent()) { - indexAclChangeSetAfterAsynchronous(changeSetsIndexed, state); + break; + } + + final AtomicInteger counter = new AtomicInteger(); + Collection> changeSetBatches = aclChangeSets.getAclChangeSets().stream() + .collect(Collectors.groupingBy(it -> counter.getAndAdd(it.getAclCount()) / changeSetAclsBatchSize)).values(); + + + for (List changeSetBatch : changeSetBatches) + { + aclCount = indexBatchOfChangeSets(changeSetBatch); + for (AclChangeSet indexed : changeSetBatch) + { + changeSetsFound.add(indexed); + } + // Update last committed transactions + setLastChangeSetIdAndCommitTimeInTrackerState(changeSetBatch, state); + indexAclChangeSetAfterWorker(changeSetBatch, state); + long endElapsed = System.nanoTime(); trackerStats.addElapsedAclTime(aclCount, endElapsed-startElapsed); startElapsed = endElapsed; - aclCount = 0; - } + }; + } - catch(InterruptedException e) + catch(InterruptedException | ExecutionException e) { throw new IOException(e); } @@ -781,32 +801,43 @@ public class AclTracker extends AbstractTracker } while ((aclChangeSets.getAclChangeSets().size() > 0) && (upToDate == false)); - LOGGER.info("{}-[CORE {}] Tracked {} ACLs", Thread.currentThread().getId(), coreName, totalAclCount); - + LOGGER.info("{}-[CORE {}] Tracked {} ACLs", Thread.currentThread().getId(), coreName, totalAclCount); + } - private void setLastChangeSetIdAndCommitTimeInTrackerState(AclChangeSets aclChangeSets, TrackerState state) + private void setLastChangeSetIdAndCommitTimeInTrackerState(List aclChangeSets, TrackerState state) { - Long maxChangeSetCommitTime = aclChangeSets.getMaxChangeSetCommitTime(); - if(maxChangeSetCommitTime != null) + + if (!aclChangeSets.isEmpty()) { - state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTime); - } - - Long maxChangeSetId = aclChangeSets.getMaxChangeSetId(); - if(maxChangeSetId != null) - { - state.setLastChangeSetIdOnServer(maxChangeSetId); + Long maxChangeSetCommitTime = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getCommitTimeMs)).get().getCommitTimeMs(); + if(maxChangeSetCommitTime != null) + { + state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTime); + } + + Long maxChangeSetId = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getId)).get().getId(); + if(maxChangeSetId != null) + { + state.setLastChangeSetIdOnServer(maxChangeSetId); + } } + } - private void indexAclChangeSetAfterAsynchronous(HashSet changeSetsIndexed, TrackerState state) + /** + * Index ACL Change Set transaction after ACLs has been indexed by the worker + * + * @param changeSetsIndexed List of ACL Change Sets indexed + * @param state + * @throws IOException + */ + private void indexAclChangeSetAfterWorker(Collection changeSetsIndexed, TrackerState state) throws IOException { - waitForAsynchronous(); for (AclChangeSet set : changeSetsIndexed) { - super.infoSrv.indexAclTransaction(set, true); + infoSrv.indexAclTransaction(set, true); // Acl change sets are ordered by commit time and tie-broken by id if (set.getCommitTimeMs() > state.getLastIndexedChangeSetCommitTime() || set.getCommitTimeMs() == state.getLastIndexedChangeSetCommitTime() @@ -817,8 +848,6 @@ public class AclTracker extends AbstractTracker } trackerStats.addChangeSetAcls(set.getAclCount()); } - changeSetsIndexed.clear(); - //super.infoSrv.commit(); } private int getAclCount(List changeSetBatch) @@ -831,19 +860,24 @@ public class AclTracker extends AbstractTracker return count; } - private int indexBatchOfChangeSets(List changeSetBatch) throws AuthenticationException, IOException, JSONException - { - int aclCount = 0; - ArrayList nonEmptyChangeSets = new ArrayList(changeSetBatch.size()); - for (AclChangeSet set : changeSetBatch) - { - if (set.getAclCount() > 0) - { - nonEmptyChangeSets.add(set); - } - } + /** + * Index ACLs from ACL Change Sets contained in changeSetBatch + * When total ACL indexed count is greater than the specified for a single execution + * (maxAclsPerExecution), no more ACL Change Sets are processed. + * + * @param changeSetBatch List of ACL Change Sets to be indexed + * @return List of ACL Change Set indexed and Count of ACL indexed + * + * @throws AuthenticationException + * @throws IOException + * @throws JSONException + */ + private int indexBatchOfChangeSets(List changeSetBatch) throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException { + // Exclude ACL Change Set with no ACLs inside + List nonEmptyChangeSets = changeSetBatch.stream() + .filter(set -> set.getAclCount() > 0) + .collect(Collectors.toList()); - ArrayList aclBatch = new ArrayList(); List acls = client.getAcls(nonEmptyChangeSets, null, Integer.MAX_VALUE); if (LOGGER.isDebugEnabled()) @@ -851,40 +885,29 @@ public class AclTracker extends AbstractTracker LOGGER.debug("{}-[CORE {}] Found {} Acls from Acl Change Sets: {}", Thread.currentThread().getId(), coreName, acls.size(), nonEmptyChangeSets); } + + List> aclBatches = Lists.partition(acls, aclBatchSize); - for (Acl acl : acls) - { - if (LOGGER.isTraceEnabled()) - { - LOGGER.trace("{}-[CORE {}] Adding ACL {} to scheduled indexing job", Thread.currentThread().getId(), - coreName, acl.toString()); - } - aclBatch.add(acl); - if (aclBatch.size() > aclBatchSize) - { - aclCount += aclBatch.size(); - AclIndexWorkerRunnable aiwr = new AclIndexWorkerRunnable(this.threadHandler, aclBatch); - this.threadHandler.scheduleTask(aiwr); - aclBatch = new ArrayList(); - } - } - if (aclBatch.size() > 0) - { - aclCount += aclBatch.size(); - AclIndexWorkerRunnable aiwr = new AclIndexWorkerRunnable(this.threadHandler, aclBatch); - this.threadHandler.scheduleTask(aiwr); - aclBatch = new ArrayList(); - } - return aclCount; + Integer processedAcls = forkJoinPool.submit(() -> + aclBatches.parallelStream().map(batch -> { + new AclIndexWorker(batch).run(); + return batch.size(); + }).reduce(0, Integer::sum) + ).get(); + + return processedAcls; } - class AclIndexWorkerRunnable extends AbstractWorkerRunnable + + /** + * ACL Indexer + */ + class AclIndexWorker extends AbstractWorker { List acls; - AclIndexWorkerRunnable(QueueHandler queueHandler, List acls) + AclIndexWorker(List acls) { - super(queueHandler); this.acls = acls; } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java index 076abe576..372a50079 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java @@ -25,11 +25,16 @@ import static org.alfresco.solr.utils.Utils.notNullOrEmpty; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Semaphore; +import com.google.common.collect.Lists; import org.alfresco.httpclient.AuthenticationException; import org.alfresco.solr.InformationServer; import org.alfresco.solr.client.NodeMetaData; @@ -45,14 +50,46 @@ import org.slf4j.LoggerFactory; */ public class CascadeTracker extends AbstractTracker implements Tracker { - private static final Logger LOGGER = LoggerFactory.getLogger(CascadeTracker.class); + + protected final static Logger LOGGER = LoggerFactory.getLogger(CascadeTracker.class); + + private static final int DEFAULT_CASCADE_TRACKER_MAX_PARALLELISM = 32; + private static final int DEFAULT_CASCADE_NODE_BATCH_SIZE = 10; + + + // Share run and write locks across all CascadeTracker threads + private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private int cascadeBatchSize; + private ForkJoinPool forkJoinPool; + private int cascadeTrackerParallelism; + + @Override + public Semaphore getWriteLock() + { + return WRITE_LOCK_BY_CORE.get(coreName); + } + @Override + public Semaphore getRunLock() + { + return RUN_LOCK_BY_CORE.get(coreName); + } + public CascadeTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) { super(p, client, coreName, informationServer, Tracker.Type.CASCADE); - threadHandler = new ThreadHandler(p, coreName, "CascadeTracker"); + cascadeTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.cascadeTrackerMaxParallelism", + String.valueOf(DEFAULT_CASCADE_TRACKER_MAX_PARALLELISM))); + + cascadeBatchSize = Integer.parseInt(p.getProperty("alfresco.cascadeNodeBatchSize", + String.valueOf(DEFAULT_CASCADE_NODE_BATCH_SIZE)));; + + forkJoinPool = new ForkJoinPool(cascadeTrackerParallelism); + RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } CascadeTracker() @@ -85,24 +122,22 @@ public class CascadeTracker extends AbstractTracker implements Tracker processCascades(iterationId); } - private void updateTransactionsAfterAsynchronous(List txsIndexed) + private void updateTransactionsAfterWorker(List txsIndexed) throws IOException { - waitForAsynchronous(); for (Transaction tx : txsIndexed) { super.infoSrv.updateTransaction(tx); } } - class CascadeIndexWorkerRunnable extends AbstractWorkerRunnable + class CascadeIndexWorker extends AbstractWorker { InformationServer infoServer; List nodes; - CascadeIndexWorkerRunnable(QueueHandler queueHandler, List nodes, InformationServer infoServer) + CascadeIndexWorker(List nodes, InformationServer infoServer) { - super(queueHandler); this.infoServer = infoServer; this.nodes = nodes; } @@ -116,7 +151,7 @@ public class CascadeTracker extends AbstractTracker implements Tracker @Override protected void onFail(Throwable failCausedBy) { - setRollback(true, failCausedBy); + setRollback(true, failCausedBy); } } @@ -129,58 +164,69 @@ public class CascadeTracker extends AbstractTracker implements Tracker private void processCascades(String iterationId) throws IOException { int num = 50; - List txBatch; - do - { - try - { + List txBatch = null; + long totalUpdatedDocs = 0; + + do { + try { getWriteLock().acquire(); txBatch = infoSrv.getCascades(num); - if(txBatch.size() == 0) + + if (txBatch.size() > 0) { - //No transactions to process for cascades. + LOGGER.info("{}-[CORE {}] Found {} transactions, transactions from {} to {}", + Thread.currentThread().getId(), + coreName, + txBatch.size(), + txBatch.get(0), + txBatch.get(txBatch.size() - 1)); + } + else + { + LOGGER.info("{}-[CORE {}] No transaction found", + Thread.currentThread().getId(), coreName); + } + + if(txBatch.size() == 0) { return; } ArrayList txIds = new ArrayList<>(); Set txIdSet = new HashSet<>(); - for (Transaction tx : txBatch) - { + for (Transaction tx : txBatch) { txIds.add(tx.getId()); txIdSet.add(tx.getId()); } List nodeMetaDatas = infoSrv.getCascadeNodes(txIds); + Integer processedCascades = 0; - if(nodeMetaDatas.size() > 0) - { - LinkedList stack = new LinkedList<>(); - stack.addAll(nodeMetaDatas); - int batchSize = 10; + if(nodeMetaDatas.size() > 0) { + List> nodeBatches = Lists.partition(nodeMetaDatas, cascadeBatchSize); - do - { - List batch = new ArrayList<>(); - while (batch.size() < batchSize && stack.size() > 0) - { - batch.add(stack.removeFirst()); - } + processedCascades = forkJoinPool.submit( () -> + nodeBatches.parallelStream().map( batch -> { + + CascadeIndexWorker worker = new CascadeIndexWorker(batch, infoSrv); + worker.run(); + + if (LOGGER.isTraceEnabled()) + { + String nodes = notNullOrEmpty(batch).stream() + .map(NodeMetaData::getId) + .map(Object::toString) + .collect(joining(",")); + LOGGER.trace("[{} / {} / {} / {}] Worker has been created for nodes {}", coreName, trackerId, iterationId, worker.hashCode(), nodes); + } + return batch.size(); + }).reduce(0, Integer::sum) + ).get(); - CascadeIndexWorkerRunnable worker = new CascadeIndexWorkerRunnable(this.threadHandler, batch, infoSrv); - if (LOGGER.isTraceEnabled()) - { - String nodes = notNullOrEmpty(batch).stream() - .map(NodeMetaData::getId) - .map(Object::toString) - .collect(joining(",")); - LOGGER.trace("[{} / {} / {} / {}] Worker has been created for nodes {}", coreName, trackerId, iterationId, worker.hashCode(), nodes); - } - this.threadHandler.scheduleTask(worker); - } while (stack.size() > 0); } //Update the transaction records. - updateTransactionsAfterAsynchronous(txBatch); + updateTransactionsAfterWorker(txBatch); + totalUpdatedDocs += processedCascades; } catch (AuthenticationException e) { @@ -193,11 +239,15 @@ public class CascadeTracker extends AbstractTracker implements Tracker catch(InterruptedException e) { throw new IOException(e); - } - finally + } catch (ExecutionException e) { + e.printStackTrace(); + } finally { getWriteLock().release(); } } while(txBatch.size() > 0); + + LOGGER.info("{}-[CORE {}] Updated {} DOCs", Thread.currentThread().getId(), coreName, totalUpdatedDocs); + } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java index fa2b2ee06..5fda789da 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java @@ -23,8 +23,11 @@ import static java.util.Optional.empty; import static java.util.Optional.ofNullable; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; import org.alfresco.solr.InformationServer; @@ -49,7 +52,21 @@ public class CommitTracker extends AbstractTracker private Optional cascadeTracker = empty(); private AtomicInteger rollbackCount = new AtomicInteger(0); - protected final static Logger log = LoggerFactory.getLogger(CommitTracker.class); + protected final static Logger LOGGER = LoggerFactory.getLogger(CommitTracker.class); + + // Share run and write locks across all CommitTracker threads + private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); + @Override + public Semaphore getWriteLock() + { + return WRITE_LOCK_BY_CORE.get(coreName); + } + @Override + public Semaphore getRunLock() + { + return RUN_LOCK_BY_CORE.get(coreName); + } /** * Default constructor, for testing. @@ -83,6 +100,9 @@ public class CommitTracker extends AbstractTracker commitInterval = Long.parseLong(p.getProperty("alfresco.commitInterval", "60000")); // Default: commit once per minute newSearcherInterval = Integer.parseInt(p.getProperty("alfresco.newSearcherInterval", "120000")); // Default: Open searchers every two minutes lastSearcherOpened = lastCommit = System.currentTimeMillis(); + + RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } public boolean hasMaintenance() throws Exception @@ -198,17 +218,17 @@ public class CommitTracker extends AbstractTracker // Log reasons why the rollback is performed if (aclTracker.getRollbackCausedBy() != null) { - log.warn("Rollback performed due to ACL Tracker error", aclTracker.getRollbackCausedBy()); + LOGGER.warn("Rollback performed due to ACL Tracker error", aclTracker.getRollbackCausedBy()); } if (metadataTracker.getRollbackCausedBy() != null) { - log.warn("Rollback performed due to Metadata Tracker error", metadataTracker.getRollbackCausedBy()); + LOGGER.warn("Rollback performed due to Metadata Tracker error", metadataTracker.getRollbackCausedBy()); } } catch (Exception e) { - log.error("Rollback failed", e); + LOGGER.error("Rollback failed", e); } finally { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java index 534b1d7e2..0da47bcc6 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java @@ -18,16 +18,21 @@ */ package org.alfresco.solr.tracker; -import java.io.IOException; -import java.util.List; -import java.util.Properties; - +import com.google.common.collect.Lists; import org.alfresco.solr.AlfrescoSolrDataModel.TenantAclIdDbId; import org.alfresco.solr.InformationServer; import org.alfresco.solr.client.SOLRAPIClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Semaphore; + /** * This tracker queries for docs with unclean content, and then updates them. * Similar to org.alfresco.repo.search.impl.lucene.ADMLuceneIndexerImpl @@ -37,18 +42,42 @@ import org.slf4j.LoggerFactory; public class ContentTracker extends AbstractTracker implements Tracker { - protected final static Logger log = LoggerFactory.getLogger(ContentTracker.class); - private int contentReadBatchSize; + protected final static Logger LOGGER = LoggerFactory.getLogger(ContentTracker.class); + + private static int DEFAULT_CONTENT_UPDATE_BATCH_SIZE = 2000; + private static final int DEFAULT_CONTENT_TRACKER_MAX_PARALLELISM = 32; + + private int contentTrackerParallelism; private int contentUpdateBatchSize; + // Share run and write locks across all ContentTracker threads + private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private ForkJoinPool forkJoinPool; + + @Override + public Semaphore getWriteLock() + { + return WRITE_LOCK_BY_CORE.get(coreName); + } + @Override + public Semaphore getRunLock() + { + return RUN_LOCK_BY_CORE.get(coreName); + } public ContentTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) { super(p, client, coreName, informationServer, Tracker.Type.CONTENT); - contentReadBatchSize = Integer.parseInt(p.getProperty("alfresco.contentReadBatchSize", "100")); - contentUpdateBatchSize = Integer.parseInt(p.getProperty("alfresco.contentUpdateBatchSize", "1000")); - threadHandler = new ThreadHandler(p, coreName, "ContentTracker"); + contentUpdateBatchSize = Integer.parseInt(p.getProperty("alfresco.contentUpdateBatchSize", + String.valueOf(DEFAULT_CONTENT_UPDATE_BATCH_SIZE))); + contentTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.contentTrackerMaxParallelism", + String.valueOf(DEFAULT_CONTENT_TRACKER_MAX_PARALLELISM))); + forkJoinPool = new ForkJoinPool(contentTrackerParallelism); + + RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } ContentTracker() @@ -59,51 +88,44 @@ public class ContentTracker extends AbstractTracker implements Tracker @Override protected void doTrack(String iterationId) throws Exception { - //System.out.println("############## Content Tracker doTrack()"); - try { + + try + { long startElapsed = System.nanoTime(); checkShutdown(); - final int ROWS = contentReadBatchSize; - int start = 0; long totalDocs = 0l; checkShutdown(); while (true) { try { - getWriteLock().acquire(); - List docs = this.infoSrv.getDocsWithUncleanContent(start, ROWS); - //System.out.println("####################### Unclean content: "+docs.size()+" ##############################:"+totalDocs); + List docs = this.infoSrv.getDocsWithUncleanContent(); if (docs.size() == 0) { break; } - int docsUpdatedSinceLastCommit = 0; - for (TenantAclIdDbId doc : docs) { - ContentIndexWorkerRunnable ciwr = new ContentIndexWorkerRunnable(super.threadHandler, doc, infoSrv); - super.threadHandler.scheduleTask(ciwr); - docsUpdatedSinceLastCommit++; + List> docBatches = Lists.partition(docs, contentUpdateBatchSize); + for (List batch : docBatches) { - if (docsUpdatedSinceLastCommit >= contentUpdateBatchSize) { - super.waitForAsynchronous(); - checkShutdown(); - //this.infoSrv.commit(); - long endElapsed = System.nanoTime(); - trackerStats.addElapsedContentTime(docsUpdatedSinceLastCommit, endElapsed - startElapsed); - startElapsed = endElapsed; - docsUpdatedSinceLastCommit = 0; - } - } - if (docsUpdatedSinceLastCommit > 0) { - super.waitForAsynchronous(); - checkShutdown(); - //this.infoSrv.commit(); + + Integer processedDocuments = forkJoinPool.submit(() -> + // Parallel task here, for example + batch.parallelStream().map(doc -> { + ContentIndexWorkerRunnable ciwr = new ContentIndexWorkerRunnable(doc, infoSrv); + ciwr.run(); + return 1; + }).reduce(0, Integer::sum) + ).get(); + long endElapsed = System.nanoTime(); - trackerStats.addElapsedContentTime(docsUpdatedSinceLastCommit, endElapsed - startElapsed); - } + trackerStats.addElapsedContentTime(processedDocuments, endElapsed - startElapsed); + startElapsed = endElapsed; + + }; + totalDocs += docs.size(); checkShutdown(); } @@ -113,7 +135,8 @@ public class ContentTracker extends AbstractTracker implements Tracker } } - log.info("total number of docs with content updated: " + totalDocs); + LOGGER.info("{}-[CORE {}] Total number of docs with content updated: {} ", Thread.currentThread().getId(), coreName, totalDocs); + } catch(Exception e) { @@ -134,14 +157,13 @@ public class ContentTracker extends AbstractTracker implements Tracker this.infoSrv.setCleanContentTxnFloor(-1); } - class ContentIndexWorkerRunnable extends AbstractWorkerRunnable + class ContentIndexWorkerRunnable extends AbstractWorker { InformationServer infoServer; TenantAclIdDbId doc; - ContentIndexWorkerRunnable(QueueHandler queueHandler, TenantAclIdDbId doc, InformationServer infoServer) + ContentIndexWorkerRunnable(TenantAclIdDbId doc, InformationServer infoServer) { - super(queueHandler); this.doc = doc; this.infoServer = infoServer; } @@ -150,7 +172,6 @@ public class ContentTracker extends AbstractTracker implements Tracker protected void doWork() throws Exception { checkShutdown(); - //System.out.println("################ Update doc:"+doc.dbId); this.infoServer.updateContentToIndexAndCache(doc.dbId, doc.tenant); } @@ -158,7 +179,7 @@ public class ContentTracker extends AbstractTracker implements Tracker protected void onFail(Throwable failCausedBy) { // This will be redone in future tracking operations - log.warn("Content tracker failed due to {}", failCausedBy.getMessage(), failCausedBy); + LOGGER.warn("Content tracker failed due to {}", failCausedBy.getMessage(), failCausedBy); } } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 551599fef..baf84b327 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -18,17 +18,7 @@ */ package org.alfresco.solr.tracker; -import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ConcurrentLinkedQueue; - +import com.google.common.collect.Lists; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.AuthenticationException; import org.alfresco.repo.index.shard.ShardState; @@ -49,17 +39,41 @@ import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Semaphore; +import java.util.stream.Collectors; + +import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE; + /* * This tracks two things: transactions and metadata nodes * @author Ahmed Owian */ public class MetadataTracker extends CoreStatePublisher implements Tracker { - protected final static Logger log = LoggerFactory.getLogger(MetadataTracker.class); - private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 100; - private static final int DEFAULT_NODE_BATCH_SIZE = 10; - private int transactionDocsBatchSize = DEFAULT_TRANSACTION_DOCS_BATCH_SIZE; - private int nodeBatchSize = DEFAULT_NODE_BATCH_SIZE; + protected final static Logger LOGGER = LoggerFactory.getLogger(MetadataTracker.class); + + private static final int DEFAULT_METADATA_TRACKER_MAX_PARALLELISM = 32; + private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 2000; + private static final int DEFAULT_MAX_NUMBER_OF_TRANSACTIONS = 2000; + private static final int DEFAULT_NODE_BATCH_SIZE = 50; + private static final String DEFAULT_INITIAL_TRANSACTION_RANGE = "0-2000"; + + private int matadataTrackerParallelism; + private int transactionDocsBatchSize; + private int nodeBatchSize; + private int maxNumberOfTransactions; + private ConcurrentLinkedQueue transactionsToReindex = new ConcurrentLinkedQueue<>(); private ConcurrentLinkedQueue transactionsToIndex = new ConcurrentLinkedQueue<>(); private ConcurrentLinkedQueue transactionsToPurge = new ConcurrentLinkedQueue<>(); @@ -67,6 +81,21 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker private ConcurrentLinkedQueue nodesToIndex = new ConcurrentLinkedQueue<>(); private ConcurrentLinkedQueue nodesToPurge = new ConcurrentLinkedQueue<>(); private ConcurrentLinkedQueue queriesToReindex = new ConcurrentLinkedQueue<>(); + private ForkJoinPool forkJoinPool; + + // Share run and write locks across all MetadataTracker threads + private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); + @Override + public Semaphore getWriteLock() + { + return WRITE_LOCK_BY_CORE.get(coreName); + } + @Override + public Semaphore getRunLock() + { + return RUN_LOCK_BY_CORE.get(coreName); + } /** * Check if nextTxCommitTimeService is available in the repository. @@ -115,12 +144,22 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker InformationServer informationServer, boolean checkRepoServicesAvailability) { super(isMaster, p, client, coreName, informationServer, Tracker.Type.METADATA); - transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); - nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); - threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); + + transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", + String.valueOf(DEFAULT_TRANSACTION_DOCS_BATCH_SIZE))); + nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", + String.valueOf(DEFAULT_NODE_BATCH_SIZE))); + maxNumberOfTransactions = Integer.parseInt(p.getProperty("alfresco.maxNumberOfTransactions", String.valueOf(DEFAULT_MAX_NUMBER_OF_TRANSACTIONS))); + matadataTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.metadataTrackerMaxParallelism", + String.valueOf(DEFAULT_METADATA_TRACKER_MAX_PARALLELISM))); + + String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", DEFAULT_INITIAL_TRANSACTION_RANGE).split("-"); cascadeTrackerEnabled = informationServer.cascadeTrackingEnabled(); - String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", "0-2000").split("-"); - minTxnIdRange = new Pair(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1])); + minTxnIdRange = new Pair<>(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1])); + forkJoinPool = new ForkJoinPool(matadataTrackerParallelism); + + RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); // In order to apply performance optimizations, checking the availability of Repo Web Scripts is required. // As these services are available from ACS 6.2 @@ -134,11 +173,11 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } catch (NoSuchMethodException e) { - log.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); + LOGGER.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); } catch (Exception e) { - log.error("Checking nextTxCommitTimeService failed.", e); + LOGGER.error("Checking nextTxCommitTimeService failed.", e); } // Try invoking txIntervalCommitTime service @@ -151,12 +190,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } catch (NoSuchMethodException e) { - log.warn("txIntervalCommitTimeServiceAvailable is not available. Upgrade your ACS Repository version " + + LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. Upgrade your ACS Repository version " + "to use this feature with DB_ID_RANGE sharding: {} ", e.getMessage()); } catch (Exception e) { - log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e); + LOGGER.error("Checking txIntervalCommitTimeServiceAvailable failed.", e); } } } @@ -171,7 +210,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker @Override protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException, EncoderException { - log.debug("### MetadataTracker doTrack ###"); // MetadataTracker must wait until ModelTracker has run ModelTracker modelTracker = this.infoSrv.getAdminHandler().getTrackerRegistry().getModelTracker(); if (modelTracker != null && modelTracker.hasModels()) @@ -208,13 +246,11 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker private void trackRepository() throws IOException, AuthenticationException, JSONException, EncoderException { - log.debug("####### MetadataTracker trackRepository Start #######"); checkShutdown(); + // Check we are tracking the correct repository TrackerState state = super.getTrackerState(); - log.debug("####### MetadataTracker check CYCLE #######"); - log.debug(String.format("%s ### state: %s", coreName, state.toString())); if(state.getTrackerCycles() == 0) { //We have a new tracker state so do the checks. @@ -254,7 +290,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { state.setCheckedLastTransactionTime(true); state.setCheckedFirstTransactionTime(true); - log.info("No transactions found - no verification required"); + LOGGER.info("No transactions found - no verification required"); firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1); if (!firstTransactions.getTransactions().isEmpty()) @@ -262,7 +298,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker Transaction firstTransaction = firstTransactions.getTransactions().get(0); long firstTransactionCommitTime = firstTransaction.getCommitTimeMs(); state.setLastGoodTxCommitTimeInIndex(firstTransactionCommitTime); - setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state); + setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions); } } @@ -286,7 +322,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } catch (NoSuchMethodException e) { - log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, " + LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, " + "upgrade your ACS Repository version in order to use the skip transactions feature: {} ", e.getMessage()); } } @@ -305,20 +341,20 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker if (setSize == 0) { - log.error("First transaction was not found with the correct timestamp."); - log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); - log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); - log.error("You can also check your SOLR connection details in solrcore.properties."); + LOGGER.error("First transaction was not found with the correct timestamp."); + LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); + LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); + LOGGER.error("You can also check your SOLR connection details in solrcore.properties."); throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp"); } else if (setSize == 1) { state.setCheckedFirstTransactionTime(true); - log.info("Verified first transaction and timestamp in index"); + LOGGER.info("Verified first transaction and timestamp in index"); } else { - log.warn("Duplicate initial transaction found with correct timestamp"); + LOGGER.warn("Duplicate initial transaction found with correct timestamp"); } } } @@ -332,7 +368,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1); } - setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state); + setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions); Long maxTxnCommitTimeInRepo = firstTransactions.getMaxTxnCommitTime(); Long maxTxnIdInRepo = firstTransactions.getMaxTxnId(); if (maxTxnCommitTimeInRepo != null && maxTxnIdInRepo != null) @@ -340,19 +376,19 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker Transaction maxTxInIndex = this.infoSrv.getMaxTransactionIdAndCommitTimeInIndex(); if (maxTxInIndex.getCommitTimeMs() > maxTxnCommitTimeInRepo) { - log.error("Last transaction was found in index with timestamp later than that of repository."); - log.error("Max Tx In Index: " + maxTxInIndex.getId() + ", In Repo: " + maxTxnIdInRepo); - log.error("Max Tx Commit Time In Index: " + maxTxInIndex.getCommitTimeMs() + ", In Repo: " + LOGGER.error("Last transaction was found in index with timestamp later than that of repository."); + LOGGER.error("Max Tx In Index: " + maxTxInIndex.getId() + ", In Repo: " + maxTxnIdInRepo); + LOGGER.error("Max Tx Commit Time In Index: " + maxTxInIndex.getCommitTimeMs() + ", In Repo: " + maxTxnCommitTimeInRepo); - log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); - log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); - log.error("You can also check your SOLR connection details in solrcore.properties."); + LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); + LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); + LOGGER.error("You can also check your SOLR connection details in solrcore.properties."); throw new AlfrescoRuntimeException("Last transaction found in index with incorrect timestamp"); } else { state.setCheckedLastTransactionTime(true); - log.info("Verified last transaction timestamp in index less than or equal to that of repository."); + LOGGER.info("Verified last transaction timestamp in index less than or equal to that of repository."); } } } @@ -390,9 +426,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker for (Node node : nodes) { docCount++; - if (log.isDebugEnabled()) + if (LOGGER.isDebugEnabled()) { - log.debug(node.toString()); + LOGGER.debug(node.toString()); } this.infoSrv.indexNode(node, false); checkShutdown(); @@ -400,14 +436,15 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // Index the transaction doc after the node - if this is not found then a reindex will be done. this.infoSrv.indexTransaction(info, false); - log.info("INDEX ACTION - Transaction {} has been indexed", transactionId); + LOGGER.info("INDEX ACTION - Transaction {} has been indexed", transactionId); requiresCommit = true; trackerStats.addTxDocs(nodes.size()); + } else { - log.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId); + LOGGER.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId); } } @@ -416,7 +453,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker if(this.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) { checkShutdown(); - //this.infoSrv.commit(); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed-startElapsed); startElapsed = endElapsed; @@ -448,7 +484,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker node.setTxnId(Long.MAX_VALUE); this.infoSrv.indexNode(node, false); - log.info("INDEX ACTION - Node {} has been reindexed", node.getId()); + LOGGER.info("INDEX ACTION - Node {} has been reindexed", node.getId()); requiresCommit = true; } checkShutdown(); @@ -489,9 +525,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker for (Node node : nodes) { docCount++; - if (log.isDebugEnabled()) + if (LOGGER.isDebugEnabled()) { - log.debug(node.toString()); + LOGGER.debug(node.toString()); } this.infoSrv.indexNode(node, true); checkShutdown(); @@ -499,11 +535,11 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // Index the transaction doc after the node - if this is not found then a reindex will be done. this.infoSrv.indexTransaction(info, true); - log.info("REINDEX ACTION - Transaction {} has been reindexed", transactionId); + LOGGER.info("REINDEX ACTION - Transaction {} has been reindexed", transactionId); } else { - log.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId); + LOGGER.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId); } } @@ -546,7 +582,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker node.setTxnId(Long.MAX_VALUE); this.infoSrv.indexNode(node, true); - log.info("REINDEX ACTION - Node {} has been reindexed", node.getId()); + LOGGER.info("REINDEX ACTION - Node {} has been reindexed", node.getId()); requiresCommit = true; } checkShutdown(); @@ -568,7 +604,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker if (query != null) { this.infoSrv.reindexNodeByQuery(query); - log.info("REINDEX ACTION - Nodes from query {} have been reindexed", query); + LOGGER.info("REINDEX ACTION - Nodes from query {} have been reindexed", query); requiresCommit = true; } checkShutdown(); @@ -593,7 +629,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // make sure it is cleaned out so we do not miss deletes this.infoSrv.deleteByTransactionId(transactionId); requiresCommit = true; - log.info("PURGE ACTION - Purged transactionId {}", transactionId); + LOGGER.info("PURGE ACTION - Purged transactionId {}", transactionId); } checkShutdown(); } @@ -601,7 +637,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker if(requiresCommit) { checkShutdown(); - //this.infoSrv.commit(); } } @@ -614,12 +649,34 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { // make sure it is cleaned out so we do not miss deletes this.infoSrv.deleteByNodeId(nodeId); - log.info("PURGE ACTION - Purged nodeId {}", nodeId); + LOGGER.info("PURGE ACTION - Purged nodeId {}", nodeId); } checkShutdown(); } } + /** + * The fromCommitTime tells getSomeTransactions() where to start, this actually fairly straight forward. + * + * What makes this code so tricky to understand is the state.getTimeToStopIndexing(). + * + * There are two scenarios to keep in mind: + * + * 1) Full re-index: In this scenario the state.getTimeToStopIndexing() will never stop the indexing. + * + * 2) Up-to-date indexing: This is where state.getTimeToStopIndexing() gets interesting. In this scenario + * the Solr index is already up to date with the repo and it is tracking new transactions. The state.getTimeToStopIndexing() + * in this scenario causes the getSomeTransactions() call to stop returning results if it finds a transaction + * beyond a specific point in time. This will break out of this loop and end the tracker run. + * + * The next time the metadata tracker runs the "continueState()" method applies the "hole retention" + * to state.getLastGoodTxCommitTimeInIndex(). This causes the state.getLastGoodTxCommitTimeInIndex() to scan + * for prior transactions that might have been missed. + * + * @param txnsFound + * @param lastGoodTxCommitTimeInIndex + * @return + */ protected Long getTxFromCommitTime(BoundedDeque txnsFound, long lastGoodTxCommitTimeInIndex) { if (txnsFound.size() > 0) { @@ -662,10 +719,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // step forward in time until we find something or hit the time bound // max id unbounded Long startTime = fromCommitTime == null ? 0L : fromCommitTime; - log.debug(String.format("#### %s MetadataTracker getSomeTransactions start time: %d end: %d", - this.coreName, - startTime, - endTime)); if(startTime == 0) { return client.getTransactions(startTime, @@ -675,6 +728,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker maxResults, shardstate); } + do { transactions = client.getTransactions(startTime, null, startTime + actualTimeStep, null, maxResults, shardstate); @@ -686,7 +740,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker Long nextTxCommitTime = client.getNextTxCommitTime(coreName, startTime); if (nextTxCommitTime != -1) { - log.info("Advancing transactions from {} to {}", startTime, nextTxCommitTime); + LOGGER.info("{}-[CORE {}] Advancing transactions from {} to {}", + Thread.currentThread().getId(), coreName, startTime, nextTxCommitTime); transactions = client.getTransactions(nextTxCommitTime, null, nextTxCommitTime + actualTimeStep, null, maxResults, shardstate); } } @@ -697,7 +752,141 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker return transactions; } + + /** + * When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction + * for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime. + * When there isn't nodes in that range, -1 is returned as commit times + * + * @param fromCommitTime Starting commit time to get transactions from Repository + * @param txnsFound List of transactions previously found + * @return List of transactions to be indexed + * + * @throws NoSuchMethodException + * @throws AuthenticationException + * @throws IOException + * @throws JSONException + * @throws EncoderException + */ + private Transactions getDBIDRangeTransactions(Long fromCommitTime, BoundedDeque txnsFound) + throws NoSuchMethodException, AuthenticationException, IOException, JSONException, EncoderException + { + boolean shardOutOfRange = false; + + DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter; + Pair commitTimes = client.getTxIntervalCommitTime(coreName, + dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange()); + Long shardMinCommitTime = commitTimes.getFirst(); + Long shardMaxCommitTime = commitTimes.getSecond(); + // Node Range it's not still available in repository + if (shardMinCommitTime == -1) + { + LOGGER.debug( + "{}-[CORE {}] [DB_ID_RANGE] No nodes in range [{}-{}] " + + "exist in the repository. Indexing only latest transaction.", + Thread.currentThread().getId(), coreName, dbIdRangeRouter.getStartRange(), + dbIdRangeRouter.getEndRange()); + shardOutOfRange = true; + } + if (fromCommitTime > shardMaxCommitTime) + { + LOGGER.debug( + "{}-[CORE {}] [DB_ID_RANGE] Last commit time is greater that max commit time in in range [{}-{}]. " + + "Indexing only latest transaction.", + Thread.currentThread().getId(), coreName, dbIdRangeRouter.getStartRange(), + dbIdRangeRouter.getEndRange()); + shardOutOfRange = true; + } + // Initial commit time for Node Range is greater than calculated from commit time + if (fromCommitTime < shardMinCommitTime) + { + LOGGER.debug("{}-[CORE {}] [DB_ID_RANGE] Skipping transactions from {} to {}", + Thread.currentThread().getId(), coreName, fromCommitTime, shardMinCommitTime); + fromCommitTime = shardMinCommitTime; + } + + Transactions transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, + state.getTimeToStopIndexing()); + + + // When transactions are out of Shard range, only the latest transaction needs to be indexed + // in order to preserve the state up-to-date of the MetadataTracker + if (shardOutOfRange) + { + Transaction latestTransaction = new Transaction(); + latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime()); + latestTransaction.setId(transactions.getMaxTxnId()); + transactions = new Transactions( + Arrays.asList(latestTransaction), + transactions.getMaxTxnCommitTime(), + transactions.getMaxTxnId()); + } + + return transactions; + } + + /** + * Remove transactions already present in SOLR index + * + * @param transactions List of transactions to be indexed + * @return List of transactions not indexed in SOLR index + */ + private Transactions removeIndexedTransactions(Transactions transactions) + { + return new Transactions(transactions.getTransactions().stream() + .filter(transaction -> { + try + { + boolean isInIndex = (transaction.getCommitTimeMs() <= state.getLastIndexedTxCommitTime() && + infoSrv.txnInIndex(transaction.getId(), true)); + if (LOGGER.isTraceEnabled()) + { + LOGGER.trace("{}-[CORE {}] Skipping Transaction Id {} as it was already indexed", + Thread.currentThread().getId(), coreName, transaction.getId()); + } + return !isInIndex; + } + catch (IOException e) + { + LOGGER.warn( + "{}-[CORE {}] Error catched while checking if Transaction Id {} was in index", + Thread.currentThread().getId(), coreName, transaction.getId(), e); + return true; + } + }) + .collect(Collectors.toList())); + + } + + + //fixme remove + /** + * Keep only transactions previous to node transaction Id + * + * @param transactions List of transactions from Repository + * @param node Last Node indexed in the cycle + * @return Filtered list of transactions + */ + private Transactions filterTransactionsByNode(Transactions transactions, Node node) + { + return new Transactions(transactions.getTransactions().stream() + .filter(transaction -> { + return transaction.getId() < node.getTxnId(); + }) + .collect(Collectors.toList())); + } + + /** + * Indexing new transactions from repository in batches of "transactionDocsBatchSize" size. + * + * Additionally, the nodes inside a transaction batch are indexed in batches of "nodeBatchSize" size. + * + * @throws AuthenticationException + * @throws IOException + * @throws JSONException + * @throws EncoderException + */ protected void trackTransactions() throws AuthenticationException, IOException, JSONException, EncoderException { long startElapsed = System.nanoTime(); @@ -705,19 +894,19 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker boolean upToDate = false; Transactions transactions; BoundedDeque txnsFound = new BoundedDeque(100); - HashSet txsIndexed = new LinkedHashSet<>(); - long totalUpdatedDocs = 0; - int docCount = 0; + int totalUpdatedDocs = 0; + + LOGGER.info("{}-[CORE {}] Starting metadata tracker execution", Thread.currentThread().getId(), coreName); do { + try { /* * This write lock is used to lock out the Commit Tracker. The ensures that the MetaDataTracker will * not be indexing content while commits or rollbacks are occurring. */ - getWriteLock().acquire(); /* @@ -725,184 +914,94 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker * tracker state could have been invalidated due to a rollback by the CommitTracker. * In this case the state will revert to the last transaction state record in the index. */ - this.state = getTrackerState(); - - /* - * The fromCommitTime tells getSomeTransactions() where to start, this actually fairly straight forward. - * - * What makes this code so tricky to understand is the state.getTimeToStopIndexing(). - * - * There are two scenarios to keep in mind: - * - * 1) Full re-index: In this scenario the state.getTimeToStopIndexing() will never stop the indexing. - * - * 2) Up-to-date indexing: This is where state.getTimeToStopIndexing() gets interesting. In this scenario - * the Solr index is already up to date with the repo and it is tracking new transactions. The state.getTimeToStopIndexing() - * in this scenario causes the getSomeTransactions() call to stop returning results if it finds a transaction - * beyond a specific point in time. This will break out of this loop and end the tracker run. - * - * The next time the metadata tracker runs the "continueState()" method applies the "hole retention" - * to state.getLastGoodTxCommitTimeInIndex(). This causes the state.getLastGoodTxCommitTimeInIndex() to scan - * for prior transactions that might have been missed. - * - */ - Long fromCommitTime = getTxFromCommitTime(txnsFound, state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() : state.getLastIndexedTxCommitTime()); - log.debug("#### Check txnsFound : " + txnsFound.size()); - log.debug("======= fromCommitTime: " + fromCommitTime); - // When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction - // for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime. - // When there isn't nodes in that range, -1 is returned as commit times - boolean shardOutOfRange = false; + // Get transaction list to be indexed if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable) { - - DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter; - Pair commitTimes = client.getTxIntervalCommitTime(coreName, - dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange()); - Long shardMinCommitTime = commitTimes.getFirst(); - Long shardMaxCommitTime = commitTimes.getSecond(); - - // Node Range it's not still available in repository - if (shardMinCommitTime == -1) - { - log.debug( - "#### [DB_ID_RANGE] No nodes in range [{}-{}] " - + "exist in the repository. Indexing only latest transaction.", - dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange()); - shardOutOfRange = true; - } - if (fromCommitTime > shardMaxCommitTime) - { - log.debug("#### [DB_ID_RANGE] Last commit time is greater that max commit time in in range [{}-{}]. " - + "Indexing only latest transaction.", - dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange()); - shardOutOfRange = true; - } - // Initial commit time for Node Range is greater than calculated from commit time - if (fromCommitTime < shardMinCommitTime) - { - log.debug("#### [DB_ID_RANGE] Skipping transactions from {} to {}", - fromCommitTime, shardMinCommitTime); - fromCommitTime = shardMinCommitTime; - } + transactions = getDBIDRangeTransactions(fromCommitTime, txnsFound); } - - log.debug("#### Get txn from commit time: " + fromCommitTime); - transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, - state.getTimeToStopIndexing()); - - - // When transactions are out of Shard range, only the latest transaction needs to be indexed - // in order to preserve the state up-to-date of the MetadataTracker - if (shardOutOfRange) + else { - Transaction latestTransaction = new Transaction(); - latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime()); - latestTransaction.setId(transactions.getMaxTxnId()); - transactions = new Transactions( - Arrays.asList(latestTransaction), - transactions.getMaxTxnCommitTime(), - transactions.getMaxTxnId()); - log.debug("#### [DB_ID_RANGE] Latest transaction to be indexed {}", latestTransaction); + transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, + state.getTimeToStopIndexing()); } - - setLastTxCommitTimeAndTxIdInTrackerState(transactions, state); - - log.debug("Scanning transactions ..."); - if (transactions.getTransactions().size() > 0) { - log.info(".... from " + transactions.getTransactions().get(0)); - log.info(".... to " + transactions.getTransactions().get(transactions.getTransactions().size() - 1)); - } else { - log.info(".... none found after lastTxCommitTime " - + ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state - .getLastIndexedTxCommitTime())); + + // Remove transactions already indexed + transactions = removeIndexedTransactions(transactions); + + if (transactions.getTransactions().size() > 0) + { + LOGGER.info("{}-[CORE {}] Found {} transactions after lastTxCommitTime {}, transactions from {} to {}", + Thread.currentThread().getId(), + coreName, + transactions.getTransactions().size(), + fromCommitTime, + transactions.getTransactions().get(0), + transactions.getTransactions().get(transactions.getTransactions().size() - 1)); + } + else + { + LOGGER.info("{}-[CORE {}] No transaction found after lastTxCommitTime {}", + Thread.currentThread().getId(), + coreName, + ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state.getLastIndexedTxCommitTime())); } - - ArrayList txBatch = new ArrayList<>(); + + // Group the transactions in batches of transactionDocsBatchSize (or less) + List> txBatches = new ArrayList<>(); + List txBatch = new ArrayList<>(); for (Transaction info : transactions.getTransactions()) { - - /* - * isInIndex is used to ensure transactions that are being re-pulled due to "hole retention" are not re-indexed if - * they have already been indexed. - * - * The logic in infoSrv.txnInIndex() first checks an in-memory LRUcache for the txnId. If it doesn't find it in the cache - * it checks the index. The LRUCache is only needed for txnId's that have been indexed but are not yet visible in the index for - * one of two reasons: - * - * 1) The commit tracker has not yet committed the transaction. - * 2) The txnId has been committed to the index but the new searcher has not yet been warmed. - * - * This means that to ensure txnId's are not needlessly reprocessed during hole retention, the LRUCache must be large - * enough to cover the time between when a txnId is indexed and when it becomes visible. - */ - - boolean isInIndex = (infoSrv.txnInIndex(info.getId(), true) && info.getCommitTimeMs() <= state.getLastIndexedTxCommitTime()); - if (isInIndex) { - txnsFound.add(info); - } else { - // Make sure we do not go ahead of where we started - we will check the holes here - // correctly next time - if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) { - upToDate = true; - break; - } - - txBatch.add(info); - if (getUpdateAndDeleteCount(txBatch) > this.transactionDocsBatchSize) { - - docCount += indexBatchOfTransactions(txBatch); - totalUpdatedDocs += docCount; - - for (Transaction scheduledTx : txBatch) { - txnsFound.add(scheduledTx); - txsIndexed.add(scheduledTx); - } - txBatch.clear(); - } + + if (LOGGER.isTraceEnabled()) + { + LOGGER.trace("{}-[CORE {}] Tracking {} Transactions. Current Transaction Id to be indexed: {}", + Thread.currentThread().getId(), coreName, transactions.getTransactions().size(), info.getId()); } - if (docCount > batchCount) { - indexTransactionsAfterAsynchronous(txsIndexed, state); - long endElapsed = System.nanoTime(); - trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); - startElapsed = endElapsed; - docCount = 0; + // Make sure we do not go ahead of where we started - we will check the holes here + // correctly next time + if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) { + upToDate = true; + break; + } - //Release the write lock allowing the commit tracker to run. - this.getWriteLock().release(); - //Re-acquire the write lock and keep indexing. - this.getWriteLock().acquire(); + txBatch.add(info); + if (getUpdateAndDeleteCount(txBatch) > transactionDocsBatchSize) { + txBatches.add(txBatch); + txBatch = new ArrayList<>(); } - checkShutdown(); } - - // Index any remaining transactions bringing the index to a consistent state so the CommitTracker can commit if need be. - - if (!txBatch.isEmpty()) { - if (this.getUpdateAndDeleteCount(txBatch) > 0) { - docCount += indexBatchOfTransactions(txBatch); - totalUpdatedDocs += docCount; - } - - for (Transaction scheduledTx : txBatch) { - txnsFound.add(scheduledTx); - txsIndexed.add(scheduledTx); - } - txBatch.clear(); + if (!txBatch.isEmpty()) + { + txBatches.add(txBatch); } + + // Index batches of transactions and the nodes updated or deleted within the transaction + for (List batch : txBatches) + { + + // Index nodes contained in the transactions + int docCount = indexBatchOfTransactions(batch, totalUpdatedDocs); + totalUpdatedDocs += docCount; + + // Add the transactions as found to avoid processing them again in the next iteration + batch.forEach(transaction -> txnsFound.add(transaction)); - if (txsIndexed.size() > 0) { - indexTransactionsAfterAsynchronous(txsIndexed, state); + // Index the transactions + indexTransactionsAfterWorker(batch); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); startElapsed = endElapsed; - docCount = 0; } + + + + setLastTxCommitTimeAndTxIdInTrackerState(transactions); + } catch(Exception e) { @@ -916,10 +1015,14 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } while ((transactions.getTransactions().size() > 0) && (upToDate == false)); - log.debug("total number of docs with metadata updated: " + totalUpdatedDocs); + LOGGER.info("{}-[CORE {}] Tracked {} DOCs", Thread.currentThread().getId(), coreName, totalUpdatedDocs); } - private void setLastTxCommitTimeAndTxIdInTrackerState(Transactions transactions, TrackerState state) + /** + * Update latest transaction indexed in MetadataTracker state + * @param transactions List of transactions indexed + */ + private void setLastTxCommitTimeAndTxIdInTrackerState(Transactions transactions) { Long maxTxnCommitTime = transactions.getMaxTxnCommitTime(); if (maxTxnCommitTime != null) @@ -934,13 +1037,17 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } } - private void indexTransactionsAfterAsynchronous(HashSet txsIndexed, TrackerState state) + /** + * Index transactions and update state of the tracker + * @param txsIndexed List of transactions to be indexed + * @throws IOException + */ + private void indexTransactionsAfterWorker(List txsIndexed) throws IOException { - waitForAsynchronous(); for (Transaction tx : txsIndexed) { - super.infoSrv.indexTransaction(tx, true); + infoSrv.indexTransaction(tx, true); // Transactions are ordered by commit time and tie-broken by tx id if (tx.getCommitTimeMs() > state.getLastIndexedTxCommitTime() || tx.getCommitTimeMs() == state.getLastIndexedTxCommitTime() @@ -952,9 +1059,13 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker trackerStats.addTxDocs((int) (tx.getDeletes() + tx.getUpdates())); } txsIndexed.clear(); - //super.infoSrv.commit(); } + /** + * Return the number of updated and deleted nodes in a list of transactions + * @param txs List of transactions + * @return Number of updated and deleted nodes + */ private long getUpdateAndDeleteCount(List txs) { long count = 0; @@ -965,12 +1076,25 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker return count; } - private int indexBatchOfTransactions(List txBatch) throws AuthenticationException, IOException, JSONException - { - int nodeCount = 0; + /** + * Index a batch of transactions. + * + * Updated or deleted nodes from these transactions are also packed into batches in order to get + * the metadata of the nodes in smaller invocations to Repository + * + * @param txBatch Batch of transactions to be indexed + * @param indexedNodes Number of nodes indexed in this Tracker execution + * + * @return Number of nodes indexed and last node indexed + * + * @throws AuthenticationException + * @throws IOException + * @throws JSONException + */ + private int indexBatchOfTransactions(List txBatch, int indexedNodes) throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException { + // Skip transactions without modifications (updates, deletes) ArrayList nonEmptyTxs = new ArrayList<>(txBatch.size()); - GetNodesParameters gnp = new GetNodesParameters(); - ArrayList txIds = new ArrayList(); + ArrayList txIds = new ArrayList<>(); for (Transaction tx : txBatch) { if (tx.getUpdates() > 0 || tx.getDeletes() > 0) @@ -979,51 +1103,48 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker txIds.add(tx.getId()); } } - + + // Get Nodes Id properties for every transaction + GetNodesParameters gnp = new GetNodesParameters(); gnp.setTransactionIds(txIds); gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); updateShardProperty(); shardProperty.ifPresent(p -> gnp.setShardProperty(p)); - gnp.setCoreName(coreName); List nodes = client.getNodes(gnp, Integer.MAX_VALUE); - - ArrayList nodeBatch = new ArrayList<>(); - for (Node node : nodes) + + if (LOGGER.isDebugEnabled()) { - if (log.isDebugEnabled()) - { - log.debug(node.toString()); - } - nodeBatch.add(node); - if (nodeBatch.size() > nodeBatchSize) - { - nodeCount += nodeBatch.size(); - NodeIndexWorkerRunnable niwr = new NodeIndexWorkerRunnable(this.threadHandler, nodeBatch, this.infoSrv); - this.threadHandler.scheduleTask(niwr); - nodeBatch = new ArrayList<>(); - } + LOGGER.debug("{}-[CORE {}] Found {} Nodes to be indexed from Transactions: {}", Thread.currentThread().getId(), + coreName, nodes.size(), txIds); } - if (nodeBatch.size() > 0) - { - nodeCount += nodeBatch.size(); - NodeIndexWorkerRunnable niwr = new NodeIndexWorkerRunnable(this.threadHandler, nodeBatch, this.infoSrv); - this.threadHandler.scheduleTask(niwr); - nodeBatch = new ArrayList<>(); - } - return nodeCount; + // Group the nodes in batches of nodeBatchSize (or less) + List> nodeBatches = Lists.partition(nodes, nodeBatchSize); + + Integer processedNodes = forkJoinPool.submit(() -> + nodeBatches.parallelStream().map(batch -> { + new NodeIndexWorker(batch, infoSrv).run(); + return batch.size(); + }).reduce(0, Integer::sum)).get(); + + + + return processedNodes; } - class NodeIndexWorkerRunnable extends AbstractWorkerRunnable + + /** + * Node Indexing class running synchronously all the tracking operations. + */ + class NodeIndexWorker extends AbstractWorker { InformationServer infoServer; List nodes; - NodeIndexWorkerRunnable(QueueHandler queueHandler, List nodes, InformationServer infoServer) + NodeIndexWorker(List nodes, InformationServer infoServer) { - super(queueHandler); this.infoServer = infoServer; this.nodes = nodes; } @@ -1186,7 +1307,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention(); DO: do { - transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000, endTime); + transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, endTime); for (Transaction info : transactions.getTransactions()) { // include diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java index 7b1223be7..748a443f8 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java @@ -30,6 +30,8 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.alfresco.error.AlfrescoRuntimeException; @@ -102,6 +104,20 @@ public class ModelTracker extends AbstractTracker implements Tracker private ReentrantReadWriteLock modelLock = new ReentrantReadWriteLock(); private volatile boolean hasModels = false; private File alfrescoModelDir; + + // Share run and write locks across all ModelTracker threads + private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); + @Override + public Semaphore getWriteLock() + { + return WRITE_LOCK_BY_CORE.get(coreName); + } + @Override + public Semaphore getRunLock() + { + return RUN_LOCK_BY_CORE.get(coreName); + } public ModelTracker(String solrHome, Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) @@ -116,6 +132,9 @@ public class ModelTracker extends AbstractTracker implements Tracker } loadPersistedModels(); + + RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } @Override diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/QueueHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/QueueHandler.java deleted file mode 100644 index 72834442b..000000000 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/QueueHandler.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2005-2014 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.tracker; - -public interface QueueHandler -{ - void removeFromQueueAndProdHead(AbstractWorkerRunnable job); -} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java index 5562c34d6..02206d95e 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java @@ -30,7 +30,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Map; import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; /** * Despite belonging to the Tracker ecosystem, this component is actually a publisher, which periodically informs @@ -51,6 +54,21 @@ public class SlaveCoreStatePublisher extends CoreStatePublisher { private static final Logger LOGGER = LoggerFactory.getLogger(SlaveCoreStatePublisher.class); + + // Share run and write locks across all SlaveCoreStatePublisher threads + private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); + @Override + public Semaphore getWriteLock() + { + return WRITE_LOCK_BY_CORE.get(coreName); + } + @Override + public Semaphore getRunLock() + { + return RUN_LOCK_BY_CORE.get(coreName); + } + public SlaveCoreStatePublisher( boolean isMaster, Properties coreProperties, @@ -59,6 +77,9 @@ public class SlaveCoreStatePublisher extends CoreStatePublisher SolrInformationServer informationServer) { super(isMaster, coreProperties, repositoryClient, name, informationServer, NODE_STATE_PUBLISHER); + + RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } @Override diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ThreadHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ThreadHandler.java deleted file mode 100644 index e22abf49f..000000000 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ThreadHandler.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (C) 2005-2014 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.tracker; - -import java.util.Properties; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.alfresco.solr.tracker.pool.DefaultTrackerPoolFactory; -import org.alfresco.solr.tracker.pool.TrackerPoolFactory; - -/** - * This class handles threads for trackers. - * - * @author Ahmed Owian - */ -public class ThreadHandler implements QueueHandler -{ - /** the instance that will be given out by the factory */ - private ThreadPoolExecutor threadPool; - private LinkedBlockingQueue threadQueue = new LinkedBlockingQueue<>(); - private ReentrantReadWriteLock threadLock = new ReentrantReadWriteLock(true); - - public ThreadHandler(Properties p, String coreName, String trackerName) - { - // construct the instance - TrackerPoolFactory trackerPoolFactory = new DefaultTrackerPoolFactory(p, coreName, trackerName); - threadPool = trackerPoolFactory.create(); - } - - - public void scheduleTask(AbstractWorkerRunnable awr) - { - try - { - threadLock.writeLock().lock(); - // Add the runnable to the queue to ensure ordering - threadQueue.add(awr); - } - finally - { - threadLock.writeLock().unlock(); - } - threadPool.execute(awr); - } - - /** - * Removes the job from the queue and notifies the HEAD - */ - public void removeFromQueueAndProdHead(AbstractWorkerRunnable job) - { - try - { - threadLock.writeLock().lock(); - // Remove self from head of queue - threadQueue.remove(job); - } - finally - { - threadLock.writeLock().unlock(); - } - } - - /** - * Read-safe method to peek at the head of the queue - */ - public AbstractWorkerRunnable peekHeadReindexWorker() - { - try - { - threadLock.readLock().lock(); - return threadQueue.peek(); - } - finally - { - threadLock.readLock().unlock(); - } - } - - public void shutDownThreadPool() - { - if (threadPool != null) - { - threadPool.shutdownNow(); - } - } -} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/TrackerJob.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/TrackerJob.java index d08d7d1ce..c5e90506c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/TrackerJob.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/TrackerJob.java @@ -18,6 +18,7 @@ */ package org.alfresco.solr.tracker; +import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -28,8 +29,12 @@ import org.slf4j.LoggerFactory; * Generic Solr tracker job, allowing Quartz to initiate an index update from * a {@link Tracker} regardless of specific implementation. * + * Concurrent execution is disallowed, as no parallel work can be done when + * indexing contents from the repository. + * * @author Matt Ward */ +@DisallowConcurrentExecution public class TrackerJob implements Job { public static final String JOBDATA_TRACKER_KEY = "TRACKER"; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java index b8c91d370..ee5ac61e5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java @@ -99,7 +99,7 @@ public class DefaultTrackerPoolFactory implements TrackerPoolFactory { case "AclTracker": corePoolSize = parseConfig("alfresco.acl.tracker.corePoolSize", p, corePoolSize); - maximumPoolSize = parseConfig("alfresco.acl.tracker.maximumPoolSize", p, maximumPoolSize); + maximumPoolSize = parseConfig("alfresco.acl.tracker.maximumPoolSize", p, maximumPoolSize); keepAliveTime = parseConfig("alfresco.acl.tracker.keepAliveTime", p, keepAliveTime); threadPriority = parseConfig("alfresco.acl.tracker.threadPriority", p, threadPriority); threadDaemon = parseConfigBoolean("alfresco.acl.tracker.threadDaemon", p); diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index 6209e74cf..37336b301 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -122,12 +122,22 @@ solr.maxBooleanClauses=10000 # Batch fetch -alfresco.transactionDocsBatchSize=500 +#Max number of transactions fetched by metadata tracker +#alfresco.maxNumberOfTransactions= + +alfresco.transactionDocsBatchSize=2000 alfresco.nodeBatchSize=100 alfresco.changeSetAclsBatchSize=500 alfresco.aclBatchSize=100 alfresco.contentReadBatchSize=100 alfresco.contentUpdateBatchSize=1000 +alfresco.cascadeNodeBatchSize=10 + +# Trackers thread pools +#alfresco.metadataTrackerMaxParallelism= +#alfresco.aclTrackerMaxParallelism= +#alfresco.contentTrackerMaxParallelism= +#alfresco.cascadeTrackerMaxParallelism # Warming diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java index 7b3650462..c0e2772fa 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java @@ -850,7 +850,8 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer public static void indexTransaction(Transaction transaction, List nodes, List nodeMetaDatas) { - //First map the nodes to a transaction. + // First map the nodes to a transaction. + nodes.stream().forEach(node -> node.setTxnId(transaction.getId())); SOLRAPIQueueClient.nodeMap.put(transaction.getId(), nodes); //Next map a node to the NodeMetaData diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index d73b3b2a9..b93277994 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -1,6 +1,7 @@ package org.alfresco.solr; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.alfresco.solr.basics.RandomSupplier; import org.alfresco.solr.client.SOLRAPIQueueClient; import org.apache.commons.io.FileUtils; @@ -67,7 +68,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.createCoreUsingTemplate; * @since solr 1.5 * @author Michael Suzuki */ -@ThreadLeakLingering(linger = 5000) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) public abstract class SolrITInitializer extends SolrTestCaseJ4 { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/basics/SolrResponsesComparator.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/basics/SolrResponsesComparator.java index 83f69b318..56d27eafe 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/basics/SolrResponsesComparator.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/basics/SolrResponsesComparator.java @@ -1,5 +1,6 @@ package org.alfresco.solr.basics; +import com.google.common.collect.Sets; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; @@ -12,8 +13,10 @@ import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; public class SolrResponsesComparator { @@ -51,6 +54,7 @@ public class SolrResponsesComparator handle.put("_version_", SKIP); handle.put("_original_parameters_", SKIP); handle.put("spellcheck-extras", SKIP); // No longer used can be removed in Solr 6. + handle.put("FIELDS", UNORDERED); } @@ -85,27 +89,6 @@ public class SolrResponsesComparator public void compareResponses(QueryResponse a, QueryResponse b) { - if (System.getProperty("remove.version.field") != null) - { - // we don't care if one has a version and the other doesnt - - // control vs distrib - // TODO: this should prob be done by adding an ignore on _version_ - // rather than mutating the responses? - if (a.getResults() != null) - { - for (SolrDocument doc : a.getResults()) - { - doc.removeFields("_version_"); - } - } - if (b.getResults() != null) - { - for (SolrDocument doc : b.getResults()) - { - doc.removeFields("_version_"); - } - } - } compareSolrResponses(a, b); } @@ -233,6 +216,29 @@ public class SolrResponsesComparator return compare1(b, a, flags, handle); } + public static String compare(Set a, Set b, int flags, Map handle) + { + String cmp; + cmp = compare1(a, b, flags, handle); + if (cmp != null) + return cmp; + return compare1(b, a, flags, handle); + } + + private static String compare1(Set a, Set b, int flags, Map handle) { + for (Object valA : a) + { + int flagsa = flags(handle, valA); + if ((flagsa & SKIP) != 0) + continue; + if (!b.contains(valA)) + { + return "[" + valA + "]==null"; + } + } + return null; + } + public static String compare(SolrDocument a, SolrDocument b, int flags, Map handle) { return compare(a.getFieldValuesMap(), b.getFieldValuesMap(), flags, handle); @@ -315,10 +321,22 @@ public class SolrResponsesComparator public static String compare(Object[] a, Object[] b, int flags, Map handle) { + + boolean ordered = (flags & UNORDERED) == 0; + if (a.length != b.length) { return ".length:" + a.length + "!=" + b.length; } + + if (!ordered) + { + Set setA = Sets.newHashSet(a); + Set setB = Sets.newHashSet(b); + return compare(setA, setB, flags, handle); + } + + for (int i = 0; i < a.length; i++) { String cmp = compare(a[i], b[i], flags, handle); @@ -328,6 +346,7 @@ public class SolrResponsesComparator return null; } + public static String compare(Object a, Object b, int flags, Map handle) { if (a == b) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java index b3e713665..0b3d7a9e0 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java @@ -53,14 +53,12 @@ public class ContentTrackerIT private TrackerStats trackerStats; private int UPDATE_BATCH = 2; - private int READ_BATCH = 400; @Before public void setUp() throws Exception { doReturn("workspace://SpacesStore").when(props).getProperty(eq("alfresco.stores"), anyString()); doReturn("" + UPDATE_BATCH).when(props).getProperty(eq("alfresco.contentUpdateBatchSize"), anyString()); - doReturn("" + READ_BATCH).when(props).getProperty(eq("alfresco.contentReadBatchSize"), anyString()); when(srv.getTrackerStats()).thenReturn(trackerStats); this.contentTracker = new ContentTracker(props, repositoryClient, coreName, srv); @@ -103,14 +101,14 @@ public class ContentTrackerIT doc.tenant = "2"; docs2.add(doc); } - when(this.srv.getDocsWithUncleanContent(anyInt(), anyInt())) + when(this.srv.getDocsWithUncleanContent()) .thenReturn(docs1) .thenReturn(docs2) .thenReturn(emptyList); this.contentTracker.doTrack("anIterationId"); InOrder order = inOrder(srv); - order.verify(srv).getDocsWithUncleanContent(0, READ_BATCH); + order.verify(srv).getDocsWithUncleanContent(); /* * I had to make each bunch of calls have different parameters to prevent Mockito from incorrectly failing @@ -126,13 +124,13 @@ public class ContentTrackerIT order.verify(srv).updateContentToIndexAndCache(thirdDoc.dbId, thirdDoc.tenant); order.verify(srv).commit(); - order.verify(srv).getDocsWithUncleanContent(0 + READ_BATCH, READ_BATCH); + order.verify(srv).getDocsWithUncleanContent(); // From docs2 order.verify(srv, times(UPDATE_BATCH)).updateContentToIndexAndCache(2l, "2"); order.verify(srv).commit(); - order.verify(srv).getDocsWithUncleanContent(0 + READ_BATCH + READ_BATCH, READ_BATCH); + order.verify(srv).getDocsWithUncleanContent(); } @Test public void typeCheck() diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java index 2be51b0da..3c7e89337 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java @@ -18,6 +18,7 @@ */ package org.alfresco.solr.tracker; +import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.Acl; @@ -27,6 +28,9 @@ import org.alfresco.solr.client.Node; import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.client.Transaction; import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; @@ -42,6 +46,7 @@ import java.util.Properties; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_SOLR4_ID; +import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME; import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; @@ -95,6 +100,14 @@ public class DistributedExpandDbidRangeAlfrescoSolrTrackerIT extends AbstractAlf bulkAcls, bulkAclReaders); + //Check for the ACL state stamp. + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), BooleanClause.Occur.MUST)); + builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID, + bulkAclChangeSet.getId(), bulkAclChangeSet.getId() + 1, true, false), BooleanClause.Occur.MUST)); + BooleanQuery waitForQuery = builder.build(); + waitForDocCount(waitForQuery, 1, MAX_WAIT_TIME); + SolrQueryResponse response0 = rangeCheck(0); NamedList values0 = response0.getValues(); //{start=0,end=100,nodeCount=0,maxDbid=0,density=NaN,expand=0,expanded=false} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java index 98bb9d353..641badb0b 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java @@ -82,7 +82,7 @@ public class DefaultTrackerPoolFactoryTest public void testAclDefaultProperties() { poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "AclTracker"); - + tpe = poolFactory.create(); assertEquals(4, tpe.getCorePoolSize()); assertEquals(4, tpe.getMaximumPoolSize()); @@ -96,7 +96,7 @@ public class DefaultTrackerPoolFactoryTest properties.put("alfresco.acl.tracker.keepAliveTime", "200"); poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "AclTracker"); tpe = poolFactory.create(); - + assertEquals(30, tpe.getCorePoolSize()); assertEquals(40, tpe.getMaximumPoolSize()); assertEquals(200, tpe.getKeepAliveTime(TimeUnit.SECONDS)); @@ -120,7 +120,7 @@ public class DefaultTrackerPoolFactoryTest properties.put("alfresco.content.tracker.keepAliveTime", "201"); poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "ContentTracker"); tpe = poolFactory.create(); - + assertEquals(100, tpe.getCorePoolSize()); assertEquals(140, tpe.getMaximumPoolSize()); assertEquals(201, tpe.getKeepAliveTime(TimeUnit.SECONDS)); @@ -129,9 +129,9 @@ public class DefaultTrackerPoolFactoryTest public void testMetaDataDefaultProperties() { poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "MetadataTracker"); - + tpe = poolFactory.create(); - + assertEquals(4, tpe.getCorePoolSize()); assertEquals(4, tpe.getMaximumPoolSize()); assertEquals(120, tpe.getKeepAliveTime(TimeUnit.SECONDS)); @@ -143,9 +143,9 @@ public class DefaultTrackerPoolFactoryTest properties.put("alfresco.metadata.tracker.maximumPoolSize", "140"); properties.put("alfresco.metadata.tracker.keepAliveTime", "201"); poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "MetadataTracker"); - + tpe = poolFactory.create(); - + assertEquals(100, tpe.getCorePoolSize()); assertEquals(140, tpe.getMaximumPoolSize()); assertEquals(201, tpe.getKeepAliveTime(TimeUnit.SECONDS)); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java index a9c6ae65b..3e2d579f4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java @@ -18,6 +18,7 @@ */ package org.alfresco.solr.transformer; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.alfresco.model.ContentModel; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; @@ -34,7 +35,6 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; @@ -56,7 +56,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; import static org.alfresco.solr.AlfrescoSolrUtils.list; -@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) @SolrTestCaseJ4.SuppressSSL public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT { @@ -80,7 +80,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT { putHandleDefaults(); //Test 1: Running a simple query without invoking CachedDocTransformer, expected to see id,DBID and _version_ - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "sort", "id asc", "shards.qt", "/afts")); assertNotNull(resp); SolrDocumentList results = resp.getResults(); assertEquals("Expecting 5 rows",5, results.size()); @@ -106,7 +106,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT putHandleDefaults(); //Test 2: Running simple query with CachedDocTransformer, expected to see all fields returned - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","*,[cached]")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts", "sort", "id asc", "fl", "*,[cached]")); SolrDocument docWithAllFields = resp.getResults().get(0); assertTrue(docWithAllFields.size() > 3); @@ -128,7 +128,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT putHandleDefaults(); //Test 3: Running simple query with CachedDocTransformer, expected to see selected fields returned - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id,DBID,[cached]")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts", "sort", "id asc", "fl","id,DBID,[cached]")); assertNotNull(resp); SolrDocument docWithRequestedFields = resp.getResults().get(0); @@ -143,7 +143,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT putHandleDefaults(); //Test 4: Running simple query with CachedDocTransformer on non default fields, expected to see selected fields returned - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id, cm_title,[cached]")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","sort", "id asc", "fl","id, cm_title,[cached]")); assertNotNull(resp); SolrDocument docWithRequestedFields3 = resp.getResults().get(0); @@ -158,7 +158,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT { putHandleDefaults(); - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_name, score, [cached]")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts", "sort", "id asc", "fl","cm_name, score, [cached]")); assertNotNull(resp); SolrDocumentList results = resp.getResults(); SolrDocument docWithAllFields = results.get(0); @@ -173,7 +173,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT { putHandleDefaults(); - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, cm_created, DBID, score, [cached]")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","sort", "id asc", "fl","cm_title, cm_created, DBID, score, [cached]")); assertNotNull(resp); SolrDocumentList results = resp.getResults(); SolrDocument docWithAllFields = results.get(0); diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java index 1f1b2189f..c59a36fd7 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java @@ -1,66 +1,79 @@ -/* - * #%L - * Alfresco Solr Client - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.solr.client; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * @author Andy - * - */ -public class AclChangeSets -{ - private List aclChangeSets; - - private Long maxChangeSetCommitTime; - - private Long maxChangeSetId; - - AclChangeSets(List aclChangeSets, Long maxChangeSetCommitTime, Long maxChangeSetId) - { - this.aclChangeSets = (aclChangeSets == null ? null : new ArrayList<>(aclChangeSets)); - this.maxChangeSetCommitTime = maxChangeSetCommitTime; - this.maxChangeSetId = maxChangeSetId; - } - - public List getAclChangeSets() - { - return Collections.unmodifiableList(aclChangeSets); - } - - public Long getMaxChangeSetCommitTime() - { - return maxChangeSetCommitTime; - } - - public Long getMaxChangeSetId() - { - return maxChangeSetId; - } - -} +/* + * #%L + * Alfresco Solr Client + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.solr.client; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * @author Andy + * + */ +public class AclChangeSets +{ + private List aclChangeSets; + + private Long maxChangeSetCommitTime; + + private Long maxChangeSetId; + + AclChangeSets(List aclChangeSets, Long maxChangeSetCommitTime, Long maxChangeSetId) + { + this.aclChangeSets = (aclChangeSets == null ? null : new ArrayList<>(aclChangeSets)); + this.maxChangeSetCommitTime = maxChangeSetCommitTime; + this.maxChangeSetId = maxChangeSetId; + } + + public AclChangeSets(List aclChangeSets) + { + this.aclChangeSets = aclChangeSets; + if (!aclChangeSets.isEmpty()) + { + this.maxChangeSetCommitTime = aclChangeSets.stream() + .max(Comparator.comparing(AclChangeSet::getCommitTimeMs)).get().getCommitTimeMs(); + this.maxChangeSetId = aclChangeSets.stream() + .max(Comparator.comparing(AclChangeSet::getId)).get().getId(); + } + } + + public List getAclChangeSets() + { + return Collections.unmodifiableList(aclChangeSets); + } + + public Long getMaxChangeSetCommitTime() + { + return maxChangeSetCommitTime; + } + + public Long getMaxChangeSetId() + { + return maxChangeSetId; + } + +} diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java index 9471fba13..76f229409 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java @@ -246,7 +246,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient { //We have moved beyond this transaction. } - else if(txn.getCommitTimeMs() > toCommitTime) + else if (toCommitTime != null && txn.getCommitTimeMs() > toCommitTime) { //We have not yet reached this transaction so break out of the loop break; diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/Transactions.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/Transactions.java index ad7bdddbf..b25621e89 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/Transactions.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/Transactions.java @@ -25,6 +25,7 @@ */ package org.alfresco.solr.client; +import java.util.Comparator; import java.util.List; /** @@ -44,7 +45,19 @@ public class Transactions this.transactions = transactions; this.maxTxnCommitTime = maxTxnCommitTime; this.maxTxnId = maxTxnId; - } + } + + public Transactions(List transactions) + { + this.transactions = transactions; + if (!transactions.isEmpty()) + { + this.maxTxnCommitTime = transactions.stream() + .max(Comparator.comparing(Transaction::getCommitTimeMs)).get().getCommitTimeMs(); + this.maxTxnId = transactions.stream() + .max(Comparator.comparing(Transaction::getId)).get().getId(); + } + } public List getTransactions() { From 3edaf416bcc071e109de763de614cc43a331f41c Mon Sep 17 00:00:00 2001 From: Keerat Date: Fri, 1 May 2020 09:30:43 +0100 Subject: [PATCH 54/54] SEARCH-2121 Changing Test Group Name to CONFIG_ENABLED_CASCADE_TRACKER --- e2e-test/src/main/java/org/alfresco/search/TestGroup.java | 2 +- .../searchServices/cmis/SolrSearchByAspectTests.java | 2 +- .../functional/searchServices/cmis/SolrSearchByIdTests.java | 2 +- .../searchServices/cmis/SolrSearchByPathTests.java | 2 +- .../searchServices/cmis/SolrSearchInTreeTests.java | 2 +- .../searchServices/search/FacetFieldsSearchTest.java | 6 +++--- .../searchServices/search/FacetRangeSearchTest.java | 6 +++--- .../functional/searchServices/search/FacetedSearchTest.java | 2 +- .../searchServices/search/PivotFacetedSearchTest.java | 2 +- .../searchServices/search/SearchAFTSInFieldTest.java | 2 +- .../functional/searchServices/search/SearchAPATHTest.java | 6 +++--- .../search/SearchSecondaryAssociationTest.java | 2 +- .../search/functional/searchServices/search/SearchTest.java | 4 ++-- .../functional/searchServices/search/StatsSearchTest.java | 2 +- .../search/tracker/CascadingIntegrationTest.java | 4 ++-- 15 files changed, 23 insertions(+), 23 deletions(-) diff --git a/e2e-test/src/main/java/org/alfresco/search/TestGroup.java b/e2e-test/src/main/java/org/alfresco/search/TestGroup.java index f08879c17..5155b8bd5 100644 --- a/e2e-test/src/main/java/org/alfresco/search/TestGroup.java +++ b/e2e-test/src/main/java/org/alfresco/search/TestGroup.java @@ -34,7 +34,7 @@ public class TestGroup public static final String CONFIG_SHARDING_EXPLICIT ="CONFIG_Sharding_EXPLICIT"; // Alfresco search services using sharded environment and explicit routing public static final String CONFIG_SHARDING_DB_ID_RANGE = "CONFIG_Sharding_DB_ID_RANGE"; // Alfresco Search Services using Sharding with DB_ID_RANGE - public static final String CONFIG_DISABLE_CASCADE_TRACKER ="Config_Disable_Cascade_Tracker"; // Alfresco search services does not index fields related to cascaded updates + public static final String CONFIG_ENABLED_CASCADE_TRACKER ="Config_Enabled_Cascade_Tracker"; // Alfresco search services does not index fields related to cascaded updates public static final String NOT_INSIGHT_ENGINE = "Not_InsightEngine"; // When Alfresco Insight Engine 1.0 isn't running diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java index 91ab34efb..0fe47d474 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java @@ -141,7 +141,7 @@ public class SolrSearchByAspectTests extends AbstractCmisE2ETest Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); } - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @XMLDataConfig(file = "src/test/resources/testdata/search-by-aspect-in-tree.xml") public void executeSearchByAspectInTree(QueryModel query) throws Exception { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java index bb5299211..3a52c75dd 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java @@ -102,7 +102,7 @@ public class SolrSearchByIdTests extends AbstractCmisE2ETest Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); } - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @XMLDataConfig(file = "src/test/resources/testdata/search-by-id-in-tree.xml") public void executeSearchByIdInTree(QueryModel query) throws Exception { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java index c81373894..a9dc4e4a7 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java @@ -49,7 +49,7 @@ public class SolrSearchByPathTests extends AbstractCmisE2ETest Utility.waitToLoopTime(getSolrWaitTimeInSeconds()); } - @Test(dependsOnMethods = "prepareDataForSearchByPath", dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(dependsOnMethods = "prepareDataForSearchByPath", dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @XMLDataConfig(file = "src/test/resources/testdata/search-by-path.xml") public void executeSearchByPathQueries(QueryModel query) { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java index c279ef57d..1be2a268e 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java @@ -56,7 +56,7 @@ public class SolrSearchInTreeTests extends AbstractCmisE2ETest dataContent.deleteSite(testSite); } - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @XMLDataConfig(file = "src/test/resources/testdata/search-in-tree.xml") public void executeCMISQuery(QueryModel query) throws Exception { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java index 597722227..f6bf50849 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java @@ -95,7 +95,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest waitForContentIndexing(htmlFile.getContent(), true); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) public void testSearchFacetFieldsBucketExcludedWhenMinCount2() throws Exception { // Create Query with FacetFields: Site and Content MimeType @@ -131,7 +131,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) public void testSearchWithFacetFieldsMinCountChecks() throws Exception { SearchRequest query = new SearchRequest(); @@ -206,7 +206,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest Assert.assertNull(response.getContext().getFacetsFields()); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) public void testSearchWithFacetFieldsOnlyFacetsWhereAccess() throws Exception { SearchRequest query = new SearchRequest(); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java index a44644c43..8614deb4d 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java @@ -109,7 +109,7 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "gap")); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") @SuppressWarnings("unchecked") @@ -159,7 +159,7 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest assertEquals(info.get("endInclusive"), "true"); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Check date facet intervals search api") @SuppressWarnings("unchecked") @@ -257,7 +257,7 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest query.setRanges(ranges); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") @SuppressWarnings("unchecked") diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java index 650be58a9..0da2b5ebc 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java @@ -102,7 +102,7 @@ public class FacetedSearchTest extends AbstractSearchServicesE2ETest waitForContentIndexing(file4.getContent(), true); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @TestRail(section = { TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Checks facet queries for the Search api") public void searchWithQueryFaceting() throws Exception { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java index 21963b825..8bba9f7b0 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java @@ -119,7 +119,7 @@ public class PivotFacetedSearchTest extends AbstractSearchServicesE2ETest assertPivotResponse(response, "creator", null); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Checks nested pivot using Search api") diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java index ff89543fb..61ec2fdf3 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java @@ -155,7 +155,7 @@ public class SearchAFTSInFieldTest extends AbstractE2EFunctionalTest }); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) public void testSearchInFieldSITE() { String query = "SITE:" + testSite.getId(); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java index 93c67af45..95df381d7 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java @@ -93,7 +93,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest *}} * */ - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) public void searchLevel0() { SearchRequest searchQuery = searchRequestWithAPATHFacet("name:*", "0"); @@ -109,7 +109,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest * Test to test that the facet buckets are returned correctly for sub-level 1/ * Test to search for a searchString, that's unique to the test run and hence stable for any environment */ - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) public void searchLevel0andIncludeSubLevel1() { SearchRequest searchQuery = searchRequestWithAPATHFacet("name:" + unique_searchString, "1/"); @@ -121,7 +121,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest getFirstBucket(response).assertThat().field("label").contains("1/"); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) public void searchLevel2() { String queryString = "name:cars"; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java index 58f3e4fbd..c9e0e0dcf 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java @@ -53,7 +53,7 @@ public class SearchSecondaryAssociationTest extends AbstractE2EFunctionalTest waitForMetadataIndexing(file1.getName(), true); } - @Test(priority = 1, groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(priority = 1, groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) public void testSearchPathForSecondaryAssociation() throws Exception { String queryPathFolder1 = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java index f1f8935d5..72c6ab684 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java @@ -101,7 +101,7 @@ public class SearchTest extends AbstractSearchServicesE2ETest response.getContext().assertThat().field("request").isNotEmpty(); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Tests a search request containing a sort clause.") public void searchWithOneSortClause() @@ -148,7 +148,7 @@ public class SearchTest extends AbstractSearchServicesE2ETest * The first clause has always the same value for all matches so the test makes sure the request is correctly * processed and the returned order is determined by the second clause. */ - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Tests a search request containing a sort clause.") public void searchWithTwoSortClauses() diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java index 7ba363b80..0236fb1ed 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java @@ -203,7 +203,7 @@ public class StatsSearchTest extends AbstractSearchServicesE2ETest assertTrue((Integer)metricCount.get("countValues") > count, "With the exclude filter there will be more documents than returned"); } - @Test(groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Checks errors with stats with Pivot using Search api") diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java index abd65e6bb..84570427f 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java @@ -76,7 +76,7 @@ public class CascadingIntegrationTest extends AbstractE2EFunctionalTest "failing while removing sharding model"); } - @Test(priority = 1, groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(priority = 1, groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) public void testChildPathWhenParentRenamed() throws Exception { // Create Parent folder @@ -119,7 +119,7 @@ public class CascadingIntegrationTest extends AbstractE2EFunctionalTest Assert.assertEquals(descendantCountOfOriginalName, 0, "Old path still has descendants: " + parentQuery); } - @Test(priority = 2, groups={TestGroup.CONFIG_DISABLE_CASCADE_TRACKER}) + @Test(priority = 2, groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) public void testGrandChildPathWhenGrandParentRenamed() throws Exception { // Create grand parent folder