From 37e612047bf41f48a7de9412ab3220cc8706e233 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Tue, 21 Jan 2020 16:13:59 +0100 Subject: [PATCH 1/2] Skipping transactions for DB_ID_RANGE Shard method. --- .../solr/tracker/MetadataTracker.java | 67 ++++++++++++++++++- .../alfresco/solr/client/SOLRAPIClient.java | 55 +++++++++++++-- 2 files changed, 117 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 6ddb64f2a..c1390bf3d 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -40,6 +40,7 @@ import org.alfresco.solr.client.Node.SolrApiNodeStatus; import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.client.Transaction; import org.alfresco.solr.client.Transactions; +import org.alfresco.util.Pair; import org.apache.commons.codec.EncoderException; import org.json.JSONException; import org.slf4j.Logger; @@ -73,6 +74,15 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker * {@link org.alfresco.solr.client.SOLRAPIClient#GET_NEXT_TX_COMMIT_TIME} */ private boolean nextTxCommitTimeServiceAvailable = false; + + /** + * Check if txInteravlCommitTimeService is available in the repository. + * This service returns the minimum and the maximum commit time for transactions in a node id range, + * so method sharding DB_ID_RANGE can skip transactions not relevant for the DB ID range. + * + * {@link org.alfresco.solr.client.SOLRAPIClient#GET_TX_INTERVAL_COMMIT_TIME} + */ + private boolean txIntervalCommitTimeServiceAvailable = false; public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) @@ -82,6 +92,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); + // Try invoking getNextTxCommitTime service try { client.getNextTxCommitTime(coreName, 0l); @@ -95,6 +106,23 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { log.error("Checking nextTxCommitTimeService failed.", e); } + + // Try invoking txIntervalCommitTime service + try + { + client.getTxIntervalCommitTime(coreName, 0l, 0l); + txIntervalCommitTimeServiceAvailable = true; + } + catch (NoSuchMethodException e) + { + log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, " + + "upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); + } + catch (Exception e) + { + log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e); + } + } MetadataTracker() @@ -640,9 +668,46 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker * */ - Long fromCommitTime = getTxFromCommitTime(txnsFound, state.getLastGoodTxCommitTimeInIndex()); + Long fromCommitTime = getTxFromCommitTime(txnsFound, + state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() : state.getLastIndexedTxCommitTime()); log.debug("#### Check txnsFound : " + txnsFound.size()); log.debug("======= fromCommitTime: " + fromCommitTime); + + // When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction + // for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime. + // When there isn't nodes in that range, -1 is returned as commit times + if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable) + { + + DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter; + Pair commitTimes = client.getTxIntervalCommitTime(coreName, + dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange()); + Long shardMinCommitTime = commitTimes.getFirst(); + Long shardMaxCommitTime = commitTimes.getSecond(); + + // Node Range it's not still available in repository + if (shardMinCommitTime == -1) + { + log.debug("#### [DB_ID_RANGE] No nodes in range [" + dbIdRangeRouter.getStartRange() + "-" + + dbIdRangeRouter.getEndRange() + "] " + + "exist in the repository. Skipping metadata tracking."); + return; + } + if (fromCommitTime > shardMaxCommitTime) + { + log.debug("#### [DB_ID_RANGE] Last commit time is greater that max commit time in in range [" + + dbIdRangeRouter.getStartRange() + "-" + dbIdRangeRouter.getEndRange() + "]. " + + "Skipping metadata tracking."); + return; + } + // Initial commit time for Node Range is greater than calculated from commit time + if (fromCommitTime < shardMinCommitTime) + { + log.debug("#### [DB_ID_RANGE] SKIPPING TRANSACTIONS FROM " + fromCommitTime + " TO " + + shardMinCommitTime); + fromCommitTime = shardMinCommitTime; + } + } log.debug("#### Get txn from commit time: " + fromCommitTime); transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 55762b340..1556f7cb6 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -74,7 +73,6 @@ import org.apache.commons.codec.EncoderException; import org.apache.commons.codec.net.URLCodec; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.util.DateUtil; -import org.apache.commons.io.IOUtils; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -106,7 +104,8 @@ public class SOLRAPIClient private static final String GET_CONTENT = "api/solr/textContent"; private static final String GET_MODEL = "api/solr/model"; private static final String GET_MODELS_DIFF = "api/solr/modelsdiff"; - private static final String GET_NEXT_TX_COMMIT_TIME = "api/solr/nextTransaction"; + private static final String GET_NEXT_TX_COMMIT_TIME = "api/solr/nextTransaction"; + private static final String GET_TX_INTERVAL_COMMIT_TIME = "api/solr/transactionInterval"; private static final String CHECKSUM_HEADER = "XAlfresco-modelChecksum"; @@ -1296,7 +1295,55 @@ public class SOLRAPIClient } return Long.parseLong(json.get("nextTransactionCommitTimeMs").toString()); - } + } + + /** + * Returns the minimum and the maximum commit time for transactions in a node id range. + * + * @param coreName alfresco, archive + * @param fromNodeId Id of the initial node + * @param toNodeId Id of the final node + * @return Time of the first transaction, time of the last transaction + * @throws IOException + * @throws AuthenticationException + * @throws NoSuchMethodException + */ + public Pair getTxIntervalCommitTime(String coreName, Long fromNodeId, Long toNodeId) + throws AuthenticationException, IOException, NoSuchMethodException + { + StringBuilder url = new StringBuilder(GET_TX_INTERVAL_COMMIT_TIME); + url.append("?").append("fromNodeId").append("=").append(fromNodeId); + url.append("&").append("toNodeId").append("=").append(toNodeId); + GetRequest get = new GetRequest(url.toString()); + Response response = null; + JSONObject json = null; + try + { + response = repositoryHttpClient.sendRequest(get); + if (response.getStatus() != HttpStatus.SC_OK) + { + throw new NoSuchMethodException(coreName + " - GetTxIntervalCommitTime return status is " + + response.getStatus() + " when invoking " + url); + } + + Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); + json = new JSONObject(new JSONTokener(reader)); + } + finally + { + if (response != null) + { + response.release(); + } + } + if (log.isDebugEnabled()) + { + log.debug(json.toString()); + } + + return new Pair(Long.parseLong(json.get("minTransactionCommitTimeMs").toString()), + Long.parseLong(json.get("maxTransactionCommitTimeMs").toString())); + } /* * type conversions from serialized JSON values to SOLR-consumable objects From f708f89bb22fb6a66f6321293ea7fcc9e95d2eba Mon Sep 17 00:00:00 2001 From: Tom Page Date: Wed, 22 Jan 2020 15:01:07 +0000 Subject: [PATCH 2/2] SEARCH-2067 Try fixing integration test order as alphabetical. Some of our integration tests fail if they are run after other tests. --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index 9af7a2537..f21eb44c8 100644 --- a/pom.xml +++ b/pom.xml @@ -66,6 +66,9 @@ + + alphabetical +