inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2026-09-16 18:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge master into dependabot-maven-search-services-org.alfresco-alfresco-data-model-8.87.

Browse Source
This commit is contained in:
Tom Page
2020-01-23 09:13:25 +00:00
parent 1e9e3fabc4 d5f2ea4359
commit 2d49c9cf53
3 changed files with 120 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
+3
View File
@@ -66,6 +66,9 @@
</goals>
</execution>
</executions>
<configuration>
<runOrder>alphabetical</runOrder>
</configuration>
</plugin>
</plugins>
</build>
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java
+66 -1
View File
@@ -40,6 +40,7 @@ import org.alfresco.solr.client.Node.SolrApiNodeStatus;
import org.alfresco.solr.client.SOLRAPIClient;
import org.alfresco.solr.client.Transaction;
import org.alfresco.solr.client.Transactions;
import org.alfresco.util.Pair;
import org.apache.commons.codec.EncoderException;
import org.json.JSONException;
import org.slf4j.Logger;
@@ -73,6 +74,15 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
* {@link org.alfresco.solr.client.SOLRAPIClient#GET_NEXT_TX_COMMIT_TIME}
*/
private boolean nextTxCommitTimeServiceAvailable = false;
/**
* Check if txInteravlCommitTimeService is available in the repository.
* This service returns the minimum and the maximum commit time for transactions in a node id range,
* so method sharding DB_ID_RANGE can skip transactions not relevant for the DB ID range.
*
* {@link org.alfresco.solr.client.SOLRAPIClient#GET_TX_INTERVAL_COMMIT_TIME}
*/
private boolean txIntervalCommitTimeServiceAvailable = false;
public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName,
InformationServer informationServer)
@@ -82,6 +92,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10"));
threadHandler = new ThreadHandler(p, coreName, "MetadataTracker");
// Try invoking getNextTxCommitTime service
try
{
client.getNextTxCommitTime(coreName, 0l);
@@ -95,6 +106,23 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
log.error("Checking nextTxCommitTimeService failed.", e);
}
// Try invoking txIntervalCommitTime service
try
{
client.getTxIntervalCommitTime(coreName, 0l, 0l);
txIntervalCommitTimeServiceAvailable = true;
}
catch (NoSuchMethodException e)
{
log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, "
+ "upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage());
}
catch (Exception e)
{
log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e);
}
}
MetadataTracker()
@@ -640,9 +668,46 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
*
*/
Long fromCommitTime = getTxFromCommitTime(txnsFound, state.getLastGoodTxCommitTimeInIndex());
Long fromCommitTime = getTxFromCommitTime(txnsFound,
state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() : state.getLastIndexedTxCommitTime());
log.debug("#### Check txnsFound : " + txnsFound.size());
log.debug("======= fromCommitTime: " + fromCommitTime);
// When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction
// for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime.
// When there isn't nodes in that range, -1 is returned as commit times
if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable)
{
DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter;
Pair<Long, Long> commitTimes = client.getTxIntervalCommitTime(coreName,
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
Long shardMinCommitTime = commitTimes.getFirst();
Long shardMaxCommitTime = commitTimes.getSecond();
// Node Range it's not still available in repository
if (shardMinCommitTime == -1)
{
log.debug("#### [DB_ID_RANGE] No nodes in range [" + dbIdRangeRouter.getStartRange() + "-"
+ dbIdRangeRouter.getEndRange() + "] "
+ "exist in the repository. Skipping metadata tracking.");
return;
}
if (fromCommitTime > shardMaxCommitTime)
{
log.debug("#### [DB_ID_RANGE] Last commit time is greater that max commit time in in range ["
+ dbIdRangeRouter.getStartRange() + "-" + dbIdRangeRouter.getEndRange() + "]. "
+ "Skipping metadata tracking.");
return;
}
// Initial commit time for Node Range is greater than calculated from commit time
if (fromCommitTime < shardMinCommitTime)
{
log.debug("#### [DB_ID_RANGE] SKIPPING TRANSACTIONS FROM " + fromCommitTime + " TO "
+ shardMinCommitTime);
fromCommitTime = shardMinCommitTime;
}
}
log.debug("#### Get txn from commit time: " + fromCommitTime);
transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000,
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java
+51 -4
View File
@@ -30,7 +30,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -74,7 +73,6 @@ import org.apache.commons.codec.EncoderException;
import org.apache.commons.codec.net.URLCodec;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.util.DateUtil;
import org.apache.commons.io.IOUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -106,7 +104,8 @@ public class SOLRAPIClient
private static final String GET_CONTENT = "api/solr/textContent";
private static final String GET_MODEL = "api/solr/model";
private static final String GET_MODELS_DIFF = "api/solr/modelsdiff";
private static final String GET_NEXT_TX_COMMIT_TIME = "api/solr/nextTransaction";
private static final String GET_NEXT_TX_COMMIT_TIME = "api/solr/nextTransaction";
private static final String GET_TX_INTERVAL_COMMIT_TIME = "api/solr/transactionInterval";
private static final String CHECKSUM_HEADER = "XAlfresco-modelChecksum";
@@ -1296,7 +1295,55 @@ public class SOLRAPIClient
}
return Long.parseLong(json.get("nextTransactionCommitTimeMs").toString());
}
}
/**
* Returns the minimum and the maximum commit time for transactions in a node id range.
*
* @param coreName alfresco, archive
* @param fromNodeId Id of the initial node
* @param toNodeId Id of the final node
* @return Time of the first transaction, time of the last transaction
* @throws IOException
* @throws AuthenticationException
* @throws NoSuchMethodException
*/
public Pair<Long, Long> getTxIntervalCommitTime(String coreName, Long fromNodeId, Long toNodeId)
throws AuthenticationException, IOException, NoSuchMethodException
{
StringBuilder url = new StringBuilder(GET_TX_INTERVAL_COMMIT_TIME);
url.append("?").append("fromNodeId").append("=").append(fromNodeId);
url.append("&").append("toNodeId").append("=").append(toNodeId);
GetRequest get = new GetRequest(url.toString());
Response response = null;
JSONObject json = null;
try
{
response = repositoryHttpClient.sendRequest(get);
if (response.getStatus() != HttpStatus.SC_OK)
{
throw new NoSuchMethodException(coreName + " - GetTxIntervalCommitTime return status is "
+ response.getStatus() + " when invoking " + url);
}
Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8"));
json = new JSONObject(new JSONTokener(reader));
}
finally
{
if (response != null)
{
response.release();
}
}
if (log.isDebugEnabled())
{
log.debug(json.toString());
}
return new Pair<Long, Long>(Long.parseLong(json.get("minTransactionCommitTimeMs").toString()),
Long.parseLong(json.get("maxTransactionCommitTimeMs").toString()));
}
/*
* type conversions from serialized JSON values to SOLR-consumable objects