inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2026-09-16 18:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/SEARCH-2028_SkipTransactions' into 'master'

Browse Source 
Skip long periods of time where repository is not ingesting new content.

See merge request search_discovery/insightengine!311
This commit is contained in:
Angel Borroy
2020-01-20 13:55:01 +00:00
parent c1e660319b 2ff2c47041
commit 9d477a5fcc
2 changed files with 84 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java
+37 -2
View File
@@ -63,6 +63,16 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
private ConcurrentLinkedQueue<Long> nodesToIndex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> nodesToPurge = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<String> queriesToReindex = new ConcurrentLinkedQueue<>();
/**
* Check if nextTxCommitTimeService is available in the repository.
* This service is used to find the next available transaction commit time from a given time,
* so periods of time where no document updating is happening can be skipped while getting
* pending transactions list.
*
* {@link org.alfresco.solr.client.SOLRAPIClient#GET_NEXT_TX_COMMIT_TIME}
*/
private boolean nextTxCommitTimeServiceAvailable = false;
public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName,
InformationServer informationServer)
@@ -71,6 +81,20 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100"));
nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10"));
threadHandler = new ThreadHandler(p, coreName, "MetadataTracker");
try
{
client.getNextTxCommitTime(coreName, 0l);
nextTxCommitTimeServiceAvailable = true;
}
catch (NoSuchMethodException e)
{
log.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage());
}
catch (Exception e)
{
log.error("Checking nextTxCommitTimeService failed.", e);
}
}
MetadataTracker()
@@ -518,7 +542,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
protected Transactions getSomeTransactions(BoundedDeque<Transaction> txnsFound, Long fromCommitTime, long timeStep,
int maxResults, long endTime) throws AuthenticationException, IOException, JSONException, EncoderException
int maxResults, long endTime) throws AuthenticationException, IOException, JSONException, EncoderException, NoSuchMethodException
{
long actualTimeStep = timeStep;
@@ -546,6 +570,17 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
transactions = client.getTransactions(startTime, null, startTime + actualTimeStep, null, maxResults, shardstate);
startTime += actualTimeStep;
// If no transactions are found, advance the time window to the next available transaction commit time
if (nextTxCommitTimeServiceAvailable && transactions.getTransactions().size() == 0)
{
Long nextTxCommitTime = client.getNextTxCommitTime(coreName, startTime);
if (nextTxCommitTime != -1)
{
log.info("Advancing transactions from {} to {}", startTime, nextTxCommitTime);
transactions = client.getTransactions(nextTxCommitTime, null, nextTxCommitTime + actualTimeStep, null, maxResults, shardstate);
}
}
} while (((transactions.getTransactions().size() == 0) && (startTime < endTime))
|| ((transactions.getTransactions().size() > 0) && alreadyFoundTransactions(txnsFound, transactions)));
@@ -964,7 +999,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
public IndexHealthReport checkIndex(Long toTx, Long toAclTx, Long fromTime, Long toTime)
throws IOException, AuthenticationException, JSONException, EncoderException
throws IOException, AuthenticationException, JSONException, EncoderException, NoSuchMethodException
{
// DB TX Count
long firstTransactionCommitTime = 0;
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java
+47 -2
View File
@@ -105,7 +105,8 @@ public class SOLRAPIClient
private static final String GET_NODES_URL = "api/solr/nodes";
private static final String GET_CONTENT = "api/solr/textContent";
private static final String GET_MODEL = "api/solr/model";
private static final String GET_MODELS_DIFF = "api/solr/modelsdiff";
private static final String GET_MODELS_DIFF = "api/solr/modelsdiff";
private static final String GET_NEXT_TX_COMMIT_TIME = "api/solr/nextTransaction";
private static final String CHECKSUM_HEADER = "XAlfresco-modelChecksum";
@@ -1251,7 +1252,51 @@ public class SOLRAPIClient
}
return diffs;
}
}
/**
* Returns the minimum and the maximum commit time for transactions in a node id range.
*
* @param coreName alfresco, archive
* @param fromCommitTime initial transaction commit time
* @return Time of the next transaction
* @throws IOException
* @throws AuthenticationException
* @throws NoSuchMethodException
*/
public Long getNextTxCommitTime(String coreName, Long fromCommitTime) throws AuthenticationException, IOException, NoSuchMethodException
{
StringBuilder url = new StringBuilder(GET_NEXT_TX_COMMIT_TIME);
url.append("?").append("fromCommitTime").append("=").append(fromCommitTime);
GetRequest get = new GetRequest(url.toString());
Response response = null;
JSONObject json = null;
try
{
response = repositoryHttpClient.sendRequest(get);
if (response.getStatus() != HttpStatus.SC_OK)
{
throw new NoSuchMethodException(coreName + " - GetNextTxCommitTime return status is "
+ response.getStatus() + " when invoking " + url);
}
Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8"));
json = new JSONObject(new JSONTokener(reader));
}
finally
{
if (response != null)
{
response.release();
}
}
if (log.isDebugEnabled())
{
log.debug(json.toString());
}
return Long.parseLong(json.get("nextTransactionCommitTimeMs").toString());
}
/*
* type conversions from serialized JSON values to SOLR-consumable objects