inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2025-09-17 14:21:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'fix/SEARCH-1917' into 'master'

Browse Source 
Fix/search 1917

See merge request search_discovery/insightengine!220
This commit is contained in:
Andrea Gazzarini
2019-11-12 16:50:57 +00:00
parent 0a5d7dda67 2ad47af4f8
commit 03aa75d383
9 changed files with 1597 additions and 1246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1465
search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java
View File

File diff suppressed because it is too large Load Diff

484
search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java
View File

@@ -1,484 +0,0 @@
/*
* Copyright (C) 2005 - 2016 Alfresco Software Limited
*
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr;
import org.alfresco.httpclient.AuthenticationException;
import org.alfresco.service.cmr.repository.datatype.Duration;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.tracker.*;
import org.alfresco.util.CachingDateFormat;
import org.apache.commons.codec.EncoderException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.json.JSONException;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Methods taken from AlfrescoCoreAdminHandler that deal with building reports
*/
public class HandlerReportBuilder {
/**
* Builds AclReport
* @param tracker
* @param aclid
* @return
* @throws IOException
* @throws JSONException
*/
public static NamedList<Object> buildAclReport(AclTracker tracker, Long aclid) throws IOException, JSONException
{
AclReport aclReport = tracker.checkAcl(aclid);
NamedList<Object> nr = new SimpleOrderedMap<Object>();
nr.add("Acl Id", aclReport.getAclId());
nr.add("Acl doc in index", aclReport.getIndexAclDoc());
if (aclReport.getIndexAclDoc() != null)
{
nr.add("Acl tx in Index", aclReport.getIndexAclTx());
}
return nr;
}
/**
* Builds TxReport
* @param trackerRegistry
* @param srv
* @param coreName
* @param tracker
* @param txid
* @return
* @throws AuthenticationException
* @throws IOException
* @throws JSONException
* @throws EncoderException
*/
public static NamedList<Object> buildTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, MetadataTracker tracker, Long txid)
throws AuthenticationException, IOException, JSONException, EncoderException
{
NamedList<Object> nr = new SimpleOrderedMap<Object>();
nr.add("TXID", txid);
nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, txid, txid, 0l, 0l, null, null));
NamedList<Object> nodes = new SimpleOrderedMap<Object>();
// add node reports ....
List<Node> dbNodes = tracker.getFullNodesForDbTransaction(txid);
for (Node node : dbNodes)
{
nodes.add("DBID " + node.getId(), buildNodeReport(tracker, node));
}
nr.add("txDbNodeCount", dbNodes.size());
nr.add("nodes", nodes);
return nr;
}
/**
* Builds AclTxReport
* @param trackerRegistry
* @param srv
* @param coreName
* @param tracker
* @param acltxid
* @return
* @throws AuthenticationException
* @throws IOException
* @throws JSONException
* @throws EncoderException
*/
public static NamedList<Object> buildAclTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, AclTracker tracker, Long acltxid)
throws AuthenticationException, IOException, JSONException, EncoderException
{
NamedList<Object> nr = new SimpleOrderedMap<Object>();
nr.add("TXID", acltxid);
nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, 0l, 0l, acltxid, acltxid, null, null));
NamedList<Object> nodes = new SimpleOrderedMap<Object>();
// add node reports ....
List<Long> dbAclIds = tracker.getAclsForDbAclTransaction(acltxid);
for (Long aclid : dbAclIds)
{
nodes.add("ACLID " + aclid, buildAclReport(tracker, aclid));
}
nr.add("aclTxDbAclCount", dbAclIds.size());
nr.add("nodes", nodes);
return nr;
}
/**
* Builds Node report
* @param tracker
* @param node
* @return
* @throws IOException
* @throws JSONException
*/
public static NamedList<Object> buildNodeReport(MetadataTracker tracker, Node node) throws IOException, JSONException
{
NodeReport nodeReport = tracker.checkNode(node);
NamedList<Object> nr = new SimpleOrderedMap<Object>();
nr.add("Node DBID", nodeReport.getDbid());
nr.add("DB TX", nodeReport.getDbTx());
nr.add("DB TX status", nodeReport.getDbNodeStatus().toString());
if (nodeReport.getIndexLeafDoc() != null)
{
nr.add("Leaf tx in Index", nodeReport.getIndexLeafTx());
}
if (nodeReport.getIndexAuxDoc() != null)
{
nr.add("Aux tx in Index", nodeReport.getIndexAuxTx());
}
nr.add("Indexed Node Doc Count", nodeReport.getIndexedNodeDocCount());
return nr;
}
/**
* Builds Node Report
* @param tracker
* @param dbid
* @return
* @throws IOException
* @throws JSONException
*/
public static NamedList<Object> buildNodeReport(MetadataTracker tracker, Long dbid) throws IOException, JSONException
{
NodeReport nodeReport = tracker.checkNode(dbid);
NamedList<Object> nr = new SimpleOrderedMap<Object>();
nr.add("Node DBID", nodeReport.getDbid());
nr.add("DB TX", nodeReport.getDbTx());
nr.add("DB TX status", nodeReport.getDbNodeStatus().toString());
if (nodeReport.getIndexLeafDoc() != null)
{
nr.add("Leaf tx in Index", nodeReport.getIndexLeafTx());
}
if (nodeReport.getIndexAuxDoc() != null)
{
nr.add("Aux tx in Index", nodeReport.getIndexAuxTx());
}
nr.add("Indexed Node Doc Count", nodeReport.getIndexedNodeDocCount());
return nr;
}
/**
* Builds Tracker report
* @param trackerRegistry
* @param srv
* @param coreName
* @param fromTx
* @param toTx
* @param fromAclTx
* @param toAclTx
* @param fromTime
* @param toTime
* @return
* @throws IOException
* @throws JSONException
* @throws AuthenticationException
* @throws EncoderException
*/
public static NamedList<Object> buildTrackerReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, Long fromTx, Long toTx, Long fromAclTx, Long toAclTx,
Long fromTime, Long toTime) throws IOException, JSONException, AuthenticationException, EncoderException
{
// ACL
AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class);
IndexHealthReport aclReport = aclTracker.checkIndex(toTx, toAclTx, fromTime, toTime);
NamedList<Object> ihr = new SimpleOrderedMap<Object>();
ihr.add("Alfresco version", aclTracker.getAlfrescoVersion());
ihr.add("DB acl transaction count", aclReport.getDbAclTransactionCount());
ihr.add("Count of duplicated acl transactions in the index", aclReport.getDuplicatedAclTxInIndex()
.cardinality());
if (aclReport.getDuplicatedAclTxInIndex().cardinality() > 0)
{
ihr.add("First duplicate acl tx", aclReport.getDuplicatedAclTxInIndex().nextSetBit(0L));
}
ihr.add("Count of acl transactions in the index but not the DB", aclReport.getAclTxInIndexButNotInDb()
.cardinality());
if (aclReport.getAclTxInIndexButNotInDb().cardinality() > 0)
{
ihr.add("First acl transaction in the index but not the DB", aclReport.getAclTxInIndexButNotInDb()
.nextSetBit(0L));
}
ihr.add("Count of missing acl transactions from the Index", aclReport.getMissingAclTxFromIndex()
.cardinality());
if (aclReport.getMissingAclTxFromIndex().cardinality() > 0)
{
ihr.add("First acl transaction missing from the Index", aclReport.getMissingAclTxFromIndex()
.nextSetBit(0L));
}
ihr.add("Index acl transaction count", aclReport.getAclTransactionDocsInIndex());
ihr.add("Index unique acl transaction count", aclReport.getAclTransactionDocsInIndex());
TrackerState aclState = aclTracker.getTrackerState();
ihr.add("Last indexed change set commit time", aclState.getLastIndexedChangeSetCommitTime());
Date lastChangeSetDate = new Date(aclState.getLastIndexedChangeSetCommitTime());
ihr.add("Last indexed change set commit date", CachingDateFormat.getDateFormat().format(lastChangeSetDate));
ihr.add("Last changeset id before holes", aclState.getLastIndexedChangeSetIdBeforeHoles());
// Metadata
MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class);
IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, toAclTx, fromTime, toTime);
ihr.add("DB transaction count", metaReport.getDbTransactionCount());
ihr.add("Count of duplicated transactions in the index", metaReport.getDuplicatedTxInIndex()
.cardinality());
if (metaReport.getDuplicatedTxInIndex().cardinality() > 0)
{
ihr.add("First duplicate", metaReport.getDuplicatedTxInIndex().nextSetBit(0L));
}
ihr.add("Count of transactions in the index but not the DB", metaReport.getTxInIndexButNotInDb()
.cardinality());
if (metaReport.getTxInIndexButNotInDb().cardinality() > 0)
{
ihr.add("First transaction in the index but not the DB", metaReport.getTxInIndexButNotInDb()
.nextSetBit(0L));
}
ihr.add("Count of missing transactions from the Index", metaReport.getMissingTxFromIndex().cardinality());
if (metaReport.getMissingTxFromIndex().cardinality() > 0)
{
ihr.add("First transaction missing from the Index", metaReport.getMissingTxFromIndex()
.nextSetBit(0L));
}
ihr.add("Index transaction count", metaReport.getTransactionDocsInIndex());
ihr.add("Index unique transaction count", metaReport.getTransactionDocsInIndex());
ihr.add("Index node count", metaReport.getLeafDocCountInIndex());
ihr.add("Count of duplicate nodes in the index", metaReport.getDuplicatedLeafInIndex().cardinality());
if (metaReport.getDuplicatedLeafInIndex().cardinality() > 0)
{
ihr.add("First duplicate node id in the index", metaReport.getDuplicatedLeafInIndex().nextSetBit(0L));
}
ihr.add("Index error count", metaReport.getErrorDocCountInIndex());
ihr.add("Count of duplicate error docs in the index", metaReport.getDuplicatedErrorInIndex()
.cardinality());
if (metaReport.getDuplicatedErrorInIndex().cardinality() > 0)
{
ihr.add("First duplicate error in the index", SolrInformationServer.PREFIX_ERROR
+ metaReport.getDuplicatedErrorInIndex().nextSetBit(0L));
}
ihr.add("Index unindexed count", metaReport.getUnindexedDocCountInIndex());
ihr.add("Count of duplicate unindexed docs in the index", metaReport.getDuplicatedUnindexedInIndex()
.cardinality());
if (metaReport.getDuplicatedUnindexedInIndex().cardinality() > 0)
{
ihr.add("First duplicate unindexed in the index",
metaReport.getDuplicatedUnindexedInIndex().nextSetBit(0L));
}
TrackerState metaState = metadataTracker.getTrackerState();
ihr.add("Last indexed transaction commit time", metaState.getLastIndexedTxCommitTime());
Date lastTxDate = new Date(metaState.getLastIndexedTxCommitTime());
ihr.add("Last indexed transaction commit date", CachingDateFormat.getDateFormat().format(lastTxDate));
ihr.add("Last TX id before holes", metaState.getLastIndexedTxIdBeforeHoles());
srv.addFTSStatusCounts(ihr);
return ihr;
}
/**
* Adds a core summary
* @param cname
* @param detail
* @param hist
* @param values
* @param srv
* @param report
* @throws IOException
*/
public static void addCoreSummary(TrackerRegistry trackerRegistry, String cname, boolean detail, boolean hist, boolean values,
InformationServer srv, NamedList<Object> report) throws IOException
{
NamedList<Object> coreSummary = new SimpleOrderedMap<Object>();
coreSummary.addAll((SimpleOrderedMap<Object>) srv.getCoreStats());
MetadataTracker metaTrkr = trackerRegistry.getTrackerForCore(cname, MetadataTracker.class);
TrackerState metadataTrkrState = metaTrkr.getTrackerState();
long lastIndexTxCommitTime = metadataTrkrState.getLastIndexedTxCommitTime();
long lastIndexedTxId = metadataTrkrState.getLastIndexedTxId();
long lastTxCommitTimeOnServer = metadataTrkrState.getLastTxCommitTimeOnServer();
long lastTxIdOnServer = metadataTrkrState.getLastTxIdOnServer();
Date lastIndexTxCommitDate = new Date(lastIndexTxCommitTime);
Date lastTxOnServerDate = new Date(lastTxCommitTimeOnServer);
long transactionsToDo = lastTxIdOnServer - lastIndexedTxId;
if (transactionsToDo < 0)
{
transactionsToDo = 0;
}
AclTracker aclTrkr = trackerRegistry.getTrackerForCore(cname, AclTracker.class);
TrackerState aclTrkrState = aclTrkr.getTrackerState();
long lastIndexChangeSetCommitTime = aclTrkrState.getLastIndexedChangeSetCommitTime();
long lastIndexedChangeSetId = aclTrkrState.getLastIndexedChangeSetId();
long lastChangeSetCommitTimeOnServer = aclTrkrState.getLastChangeSetCommitTimeOnServer();
long lastChangeSetIdOnServer = aclTrkrState.getLastChangeSetIdOnServer();
Date lastIndexChangeSetCommitDate = new Date(lastIndexChangeSetCommitTime);
Date lastChangeSetOnServerDate = new Date(lastChangeSetCommitTimeOnServer);
long changeSetsToDo = lastChangeSetIdOnServer - lastIndexedChangeSetId;
if (changeSetsToDo < 0)
{
changeSetsToDo = 0;
}
long nodesToDo = 0;
long remainingTxTimeMillis = 0;
if (transactionsToDo > 0)
{
// We now use the elapsed time as seen by the single thread farming out metadata indexing
double meanDocsPerTx = srv.getTrackerStats().getMeanDocsPerTx();
double meanNodeElaspedIndexTime = srv.getTrackerStats().getMeanNodeElapsedIndexTime();
nodesToDo = (long)(transactionsToDo * meanDocsPerTx);
remainingTxTimeMillis = (long) (nodesToDo * meanNodeElaspedIndexTime);
}
Date now = new Date();
Date end = new Date(now.getTime() + remainingTxTimeMillis);
Duration remainingTx = new Duration(now, end);
long remainingChangeSetTimeMillis = 0;
if (changeSetsToDo > 0)
{
// We now use the elapsed time as seen by the single thread farming out alc indexing
double meanAclsPerChangeSet = srv.getTrackerStats().getMeanAclsPerChangeSet();
double meanAclElapsedIndexTime = srv.getTrackerStats().getMeanAclElapsedIndexTime();
remainingChangeSetTimeMillis = (long) (changeSetsToDo * meanAclsPerChangeSet * meanAclElapsedIndexTime);
}
now = new Date();
end = new Date(now.getTime() + remainingChangeSetTimeMillis);
Duration remainingChangeSet = new Duration(now, end);
NamedList<Object> ftsSummary = new SimpleOrderedMap<Object>();
long remainingContentTimeMillis = 0;
srv.addFTSStatusCounts(ftsSummary);
long cleanCount = ((Long)ftsSummary.get("Node count with FTSStatus Clean")).longValue();
long dirtyCount = ((Long)ftsSummary.get("Node count with FTSStatus Dirty")).longValue();
long newCount = ((Long)ftsSummary.get("Node count with FTSStatus New")).longValue();
long nodesInIndex = ((Long)coreSummary.get("Alfresco Nodes in Index"));
long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount + newCount)/nodesInIndex : 0;;
if (dirtyCount + newCount + contentYetToSee > 0)
{
// We now use the elapsed time as seen by the single thread farming out alc indexing
double meanContentElapsedIndexTime = srv.getTrackerStats().getMeanContentElapsedIndexTime();
remainingContentTimeMillis = (long) ((dirtyCount + newCount + contentYetToSee) * meanContentElapsedIndexTime);
}
now = new Date();
end = new Date(now.getTime() + remainingContentTimeMillis);
Duration remainingContent = new Duration(now, end);
coreSummary.add("FTS",ftsSummary);
Duration txLag = new Duration(lastIndexTxCommitDate, lastTxOnServerDate);
if (lastIndexTxCommitDate.compareTo(lastTxOnServerDate) > 0)
{
txLag = new Duration();
}
long txLagSeconds = (lastTxCommitTimeOnServer - lastIndexTxCommitTime) / 1000;
if (txLagSeconds < 0)
{
txLagSeconds = 0;
}
Duration changeSetLag = new Duration(lastIndexChangeSetCommitDate, lastChangeSetOnServerDate);
if (lastIndexChangeSetCommitDate.compareTo(lastChangeSetOnServerDate) > 0)
{
changeSetLag = new Duration();
}
long changeSetLagSeconds = (lastChangeSetCommitTimeOnServer - lastIndexChangeSetCommitTime) / 1000;
if (txLagSeconds < 0)
{
txLagSeconds = 0;
}
ContentTracker contentTrkr = trackerRegistry.getTrackerForCore(cname, ContentTracker.class);
TrackerState contentTrkrState = contentTrkr.getTrackerState();
// Leave ModelTracker out of this check, because it is common
boolean aTrackerIsRunning = aclTrkrState.isRunning() || metadataTrkrState.isRunning()
|| contentTrkrState.isRunning();
coreSummary.add("Active", aTrackerIsRunning);
ModelTracker modelTrkr = trackerRegistry.getModelTracker();
TrackerState modelTrkrState = modelTrkr.getTrackerState();
coreSummary.add("ModelTracker Active", modelTrkrState.isRunning());
coreSummary.add("ContentTracker Active", contentTrkrState.isRunning());
coreSummary.add("MetadataTracker Active", metadataTrkrState.isRunning());
coreSummary.add("AclTracker Active", aclTrkrState.isRunning());
// TX
coreSummary.add("Last Index TX Commit Time", lastIndexTxCommitTime);
coreSummary.add("Last Index TX Commit Date", lastIndexTxCommitDate);
coreSummary.add("TX Lag", txLagSeconds + " s");
coreSummary.add("TX Duration", txLag.toString());
coreSummary.add("Timestamp for last TX on server", lastTxCommitTimeOnServer);
coreSummary.add("Date for last TX on server", lastTxOnServerDate);
coreSummary.add("Id for last TX on server", lastTxIdOnServer);
coreSummary.add("Id for last TX in index", lastIndexedTxId);
coreSummary.add("Approx transactions remaining", transactionsToDo);
coreSummary.add("Approx transaction indexing time remaining", remainingTx.largestComponentformattedString());
// Change set
coreSummary.add("Last Index Change Set Commit Time", lastIndexChangeSetCommitTime);
coreSummary.add("Last Index Change Set Commit Date", lastIndexChangeSetCommitDate);
coreSummary.add("Change Set Lag", changeSetLagSeconds + " s");
coreSummary.add("Change Set Duration", changeSetLag.toString());
coreSummary.add("Timestamp for last Change Set on server", lastChangeSetCommitTimeOnServer);
coreSummary.add("Date for last Change Set on server", lastChangeSetOnServerDate);
coreSummary.add("Id for last Change Set on server", lastChangeSetIdOnServer);
coreSummary.add("Id for last Change Set in index", lastIndexedChangeSetId);
coreSummary.add("Approx change sets remaining", changeSetsToDo);
coreSummary.add("Approx change set indexing time remaining",
remainingChangeSet.largestComponentformattedString());
coreSummary.add("Approx content indexing time remaining",
remainingContent.largestComponentformattedString());
// Stats
coreSummary.add("Model sync times (ms)",
srv.getTrackerStats().getModelTimes().getNamedList(detail, hist, values));
coreSummary.add("Acl index time (ms)",
srv.getTrackerStats().getAclTimes().getNamedList(detail, hist, values));
coreSummary.add("Node index time (ms)",
srv.getTrackerStats().getNodeTimes().getNamedList(detail, hist, values));
coreSummary.add("Docs/Tx", srv.getTrackerStats().getTxDocs().getNamedList(detail, hist, values));
coreSummary.add("Doc Transformation time (ms)", srv.getTrackerStats().getDocTransformationTimes()
.getNamedList(detail, hist, values));
// Model
Map<String, Set<String>> modelErrors = srv.getModelErrors();
if (modelErrors.size() > 0)
{
NamedList<Object> errorList = new SimpleOrderedMap<Object>();
for (Map.Entry<String, Set<String>> modelNameToErrors : modelErrors.entrySet())
{
errorList.add(modelNameToErrors.getKey(), modelNameToErrors.getValue());
}
coreSummary.add("Model changes are not compatible with the existing data model and have not been applied",
errorList);
}
report.add(cname, coreSummary);
}
}

564
search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java Normal file
View File

@@ -0,0 +1,564 @@
/*
* Copyright (C) 2005 - 2016 Alfresco Software Limited
*
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.service.cmr.repository.datatype.Duration;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.tracker.*;
import org.alfresco.util.CachingDateFormat;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.json.JSONException;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static java.util.Optional.ofNullable;
/**
* Methods taken from AlfrescoCoreAdminHandler that deal with building reports
*/
class HandlerReportHelper
{
static NamedList<Object> buildAclReport(AclTracker tracker, Long aclid) throws JSONException
{
AclReport aclReport = tracker.checkAcl(aclid);
NamedList<Object> nr = new SimpleOrderedMap<>();
nr.add("Acl Id", aclReport.getAclId());
nr.add("Acl doc in index", aclReport.getIndexAclDoc());
if (aclReport.getIndexAclDoc() != null)
{
nr.add("Acl tx in Index", aclReport.getIndexAclTx());
}
return nr;
}
static NamedList<Object> buildTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, MetadataTracker tracker, Long txid) throws JSONException
{
NamedList<Object> nr = new SimpleOrderedMap<>();
nr.add("TXID", txid);
nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, txid, txid, 0L, 0L, null, null));
NamedList<Object> nodes = new SimpleOrderedMap<>();
// add node reports ....
List<Node> dbNodes = tracker.getFullNodesForDbTransaction(txid);
for (Node node : dbNodes)
{
nodes.add("DBID " + node.getId(), buildNodeReport(tracker, node));
}
nr.add("txDbNodeCount", dbNodes.size());
nr.add("nodes", nodes);
return nr;
}
static NamedList<Object> buildAclTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, AclTracker tracker, Long acltxid) throws JSONException
{
try {
NamedList<Object> nr = new SimpleOrderedMap<>();
nr.add("TXID", acltxid);
nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, 0L, 0L, acltxid, acltxid, null, null));
NamedList<Object> nodes = new SimpleOrderedMap<>();
// add node reports ....
List<Long> dbAclIds = tracker.getAclsForDbAclTransaction(acltxid);
for (Long aclid : dbAclIds) {
nodes.add("ACLID " + aclid, buildAclReport(tracker, aclid));
}
nr.add("aclTxDbAclCount", dbAclIds.size());
nr.add("nodes", nodes);
return nr;
}
catch (Exception exception)
{
throw new AlfrescoRuntimeException("", exception);
}
}
static NamedList<Object> buildNodeReport(MetadataTracker tracker, Node node) throws JSONException
{
NodeReport nodeReport = tracker.checkNode(node);
NamedList<Object> nr = new SimpleOrderedMap<>();
nr.add("Node DBID", nodeReport.getDbid());
nr.add("DB TX", nodeReport.getDbTx());
nr.add("DB TX status", nodeReport.getDbNodeStatus().toString());
if (nodeReport.getIndexLeafDoc() != null)
{
nr.add("Leaf tx in Index", nodeReport.getIndexLeafTx());
}
if (nodeReport.getIndexAuxDoc() != null)
{
nr.add("Aux tx in Index", nodeReport.getIndexAuxTx());
}
nr.add("Indexed Node Doc Count", nodeReport.getIndexedNodeDocCount());
return nr;
}
static NamedList<Object> buildNodeReport(CoreStatePublisher publisher, Long dbid) throws JSONException
{
NodeReport nodeReport = publisher.checkNode(dbid);
NamedList<Object> payload = new SimpleOrderedMap<>();
payload.add("Node DBID", nodeReport.getDbid());
if (publisher.isOnMasterOrStandalone())
{
ofNullable(nodeReport.getDbTx()).ifPresent(value -> payload.add("DB TX", value));
ofNullable(nodeReport.getDbNodeStatus()).map(Object::toString).ifPresent(value -> payload.add("DB TX Status", value));
ofNullable(nodeReport.getIndexLeafTx()).ifPresent(value -> payload.add("Leaf tx in Index", value));
ofNullable(nodeReport.getIndexAuxDoc()).ifPresent(value -> payload.add("Aux tx in Index", value));
}
else
{
payload.add("WARNING", "This response comes from a slave core and it contains minimal information about the node. " +
"Please consider to re-submit the same request to the corresponding Master, in order to get more information.");
}
ofNullable(nodeReport.getIndexedNodeDocCount()).ifPresent(value -> payload.add("Indexed Node Doc Count", value));
return payload;
}
/**
* Builds Tracker report
*/
static NamedList<Object> buildTrackerReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, Long fromTx, Long toTx, Long fromAclTx, Long toAclTx,
Long fromTime, Long toTime) throws JSONException
{
try
{
// ACL
AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class);
IndexHealthReport aclReport = aclTracker.checkIndex(toTx, toAclTx, fromTime, toTime);
NamedList<Object> ihr = new SimpleOrderedMap<>();
ihr.add("Alfresco version", aclTracker.getAlfrescoVersion());
ihr.add("DB acl transaction count", aclReport.getDbAclTransactionCount());
ihr.add("Count of duplicated acl transactions in the index", aclReport.getDuplicatedAclTxInIndex()
.cardinality());
if (aclReport.getDuplicatedAclTxInIndex().cardinality() > 0) {
ihr.add("First duplicate acl tx", aclReport.getDuplicatedAclTxInIndex().nextSetBit(0L));
}
ihr.add("Count of acl transactions in the index but not the DB", aclReport.getAclTxInIndexButNotInDb()
.cardinality());
if (aclReport.getAclTxInIndexButNotInDb().cardinality() > 0) {
ihr.add("First acl transaction in the index but not the DB", aclReport.getAclTxInIndexButNotInDb()
.nextSetBit(0L));
}
ihr.add("Count of missing acl transactions from the Index", aclReport.getMissingAclTxFromIndex()
.cardinality());
if (aclReport.getMissingAclTxFromIndex().cardinality() > 0) {
ihr.add("First acl transaction missing from the Index", aclReport.getMissingAclTxFromIndex()
.nextSetBit(0L));
}
ihr.add("Index acl transaction count", aclReport.getAclTransactionDocsInIndex());
ihr.add("Index unique acl transaction count", aclReport.getAclTransactionDocsInIndex());
TrackerState aclState = aclTracker.getTrackerState();
ihr.add("Last indexed change set commit time", aclState.getLastIndexedChangeSetCommitTime());
Date lastChangeSetDate = new Date(aclState.getLastIndexedChangeSetCommitTime());
ihr.add("Last indexed change set commit date", CachingDateFormat.getDateFormat().format(lastChangeSetDate));
ihr.add("Last changeset id before holes", aclState.getLastIndexedChangeSetIdBeforeHoles());
// Metadata
MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class);
IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, toAclTx, fromTime, toTime);
ihr.add("DB transaction count", metaReport.getDbTransactionCount());
ihr.add("Count of duplicated transactions in the index", metaReport.getDuplicatedTxInIndex()
.cardinality());
if (metaReport.getDuplicatedTxInIndex().cardinality() > 0) {
ihr.add("First duplicate", metaReport.getDuplicatedTxInIndex().nextSetBit(0L));
}
ihr.add("Count of transactions in the index but not the DB", metaReport.getTxInIndexButNotInDb()
.cardinality());
if (metaReport.getTxInIndexButNotInDb().cardinality() > 0) {
ihr.add("First transaction in the index but not the DB", metaReport.getTxInIndexButNotInDb()
.nextSetBit(0L));
}
ihr.add("Count of missing transactions from the Index", metaReport.getMissingTxFromIndex().cardinality());
if (metaReport.getMissingTxFromIndex().cardinality() > 0) {
ihr.add("First transaction missing from the Index", metaReport.getMissingTxFromIndex()
.nextSetBit(0L));
}
ihr.add("Index transaction count", metaReport.getTransactionDocsInIndex());
ihr.add("Index unique transaction count", metaReport.getTransactionDocsInIndex());
ihr.add("Index node count", metaReport.getLeafDocCountInIndex());
ihr.add("Count of duplicate nodes in the index", metaReport.getDuplicatedLeafInIndex().cardinality());
if (metaReport.getDuplicatedLeafInIndex().cardinality() > 0) {
ihr.add("First duplicate node id in the index", metaReport.getDuplicatedLeafInIndex().nextSetBit(0L));
}
ihr.add("Index error count", metaReport.getErrorDocCountInIndex());
ihr.add("Count of duplicate error docs in the index", metaReport.getDuplicatedErrorInIndex()
.cardinality());
if (metaReport.getDuplicatedErrorInIndex().cardinality() > 0) {
ihr.add("First duplicate error in the index", SolrInformationServer.PREFIX_ERROR
+ metaReport.getDuplicatedErrorInIndex().nextSetBit(0L));
}
ihr.add("Index unindexed count", metaReport.getUnindexedDocCountInIndex());
ihr.add("Count of duplicate unindexed docs in the index", metaReport.getDuplicatedUnindexedInIndex()
.cardinality());
if (metaReport.getDuplicatedUnindexedInIndex().cardinality() > 0) {
ihr.add("First duplicate unindexed in the index",
metaReport.getDuplicatedUnindexedInIndex().nextSetBit(0L));
}
TrackerState metaState = metadataTracker.getTrackerState();
ihr.add("Last indexed transaction commit time", metaState.getLastIndexedTxCommitTime());
Date lastTxDate = new Date(metaState.getLastIndexedTxCommitTime());
ihr.add("Last indexed transaction commit date", CachingDateFormat.getDateFormat().format(lastTxDate));
ihr.add("Last TX id before holes", metaState.getLastIndexedTxIdBeforeHoles());
srv.addFTSStatusCounts(ihr);
return ihr;
}
catch (Exception exception)
{
throw new AlfrescoRuntimeException("", exception);
}
}
static void addSlaveCoreSummary(TrackerRegistry trackerRegistry, String cname, boolean detail, boolean hist, boolean values,
InformationServer srv, NamedList<Object> report) throws IOException
{
NamedList<Object> coreSummary = new SimpleOrderedMap<>();
coreSummary.addAll((SimpleOrderedMap<Object>) srv.getCoreStats());
SlaveCoreStatePublisher statePublisher = trackerRegistry.getTrackerForCore(cname, SlaveCoreStatePublisher.class);
TrackerState trackerState = statePublisher.getTrackerState();
long lastIndexTxCommitTime = trackerState.getLastIndexedTxCommitTime();
long lastIndexedTxId = trackerState.getLastIndexedTxId();
long lastTxCommitTimeOnServer = trackerState.getLastTxCommitTimeOnServer();
long lastTxIdOnServer = trackerState.getLastTxIdOnServer();
Date lastIndexTxCommitDate = new Date(lastIndexTxCommitTime);
Date lastTxOnServerDate = new Date(lastTxCommitTimeOnServer);
long transactionsToDo = lastTxIdOnServer - lastIndexedTxId;
if (transactionsToDo < 0)
{
transactionsToDo = 0;
}
long nodesToDo = 0;
long remainingTxTimeMillis = 0;
if (transactionsToDo > 0)
{
// We now use the elapsed time as seen by the single thread farming out metadata indexing
double meanDocsPerTx = srv.getTrackerStats().getMeanDocsPerTx();
double meanNodeElaspedIndexTime = srv.getTrackerStats().getMeanNodeElapsedIndexTime();
nodesToDo = (long)(transactionsToDo * meanDocsPerTx);
remainingTxTimeMillis = (long) (nodesToDo * meanNodeElaspedIndexTime);
}
Date now = new Date();
Date end = new Date(now.getTime() + remainingTxTimeMillis);
Duration remainingTx = new Duration(now, end);
long remainingChangeSetTimeMillis = 0;
now = new Date();
end = new Date(now.getTime() + remainingChangeSetTimeMillis);
Duration remainingChangeSet = new Duration(now, end);
NamedList<Object> ftsSummary = new SimpleOrderedMap<>();
long remainingContentTimeMillis = 0;
srv.addFTSStatusCounts(ftsSummary);
long cleanCount =
ofNullable(ftsSummary.get("Node count with FTSStatus Clean"))
.map(Number.class::cast)
.map(Number::longValue)
.orElse(0L);
long dirtyCount =
ofNullable(ftsSummary.get("Node count with FTSStatus Dirty"))
.map(Number.class::cast)
.map(Number::longValue)
.orElse(0L);
long newCount =
ofNullable(ftsSummary.get("Node count with FTSStatus New"))
.map(Number.class::cast)
.map(Number::longValue)
.orElse(0L);
long nodesInIndex =
ofNullable(coreSummary.get("Alfresco Nodes in Index"))
.map(Number.class::cast)
.map(Number::longValue)
.orElse(0L);
long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount + newCount)/nodesInIndex : 0;
if (dirtyCount + newCount + contentYetToSee > 0)
{
// We now use the elapsed time as seen by the single thread farming out alc indexing
double meanContentElapsedIndexTime = srv.getTrackerStats().getMeanContentElapsedIndexTime();
remainingContentTimeMillis = (long) ((dirtyCount + newCount + contentYetToSee) * meanContentElapsedIndexTime);
}
now = new Date();
end = new Date(now.getTime() + remainingContentTimeMillis);
Duration remainingContent = new Duration(now, end);
coreSummary.add("FTS",ftsSummary);
Duration txLag = new Duration(lastIndexTxCommitDate, lastTxOnServerDate);
if (lastIndexTxCommitDate.compareTo(lastTxOnServerDate) > 0)
{
txLag = new Duration();
}
long txLagSeconds = (lastTxCommitTimeOnServer - lastIndexTxCommitTime) / 1000;
if (txLagSeconds < 0)
{
txLagSeconds = 0;
}
ModelTracker modelTrkr = trackerRegistry.getModelTracker();
TrackerState modelTrkrState = modelTrkr.getTrackerState();
coreSummary.add("ModelTracker Active", modelTrkrState.isRunning());
coreSummary.add("NodeState Publisher Active", trackerState.isRunning());
// TX
coreSummary.add("Last Index TX Commit Time", lastIndexTxCommitTime);
coreSummary.add("Last Index TX Commit Date", lastIndexTxCommitDate);
coreSummary.add("TX Lag", txLagSeconds + " s");
coreSummary.add("TX Duration", txLag.toString());
coreSummary.add("Timestamp for last TX on server", lastTxCommitTimeOnServer);
coreSummary.add("Date for last TX on server", lastTxOnServerDate);
coreSummary.add("Id for last TX on server", lastTxIdOnServer);
coreSummary.add("Id for last TX in index", lastIndexedTxId);
coreSummary.add("Approx transactions remaining", transactionsToDo);
coreSummary.add("Approx transaction indexing time remaining", remainingTx.largestComponentformattedString());
// Stats
coreSummary.add("Model sync times (ms)", srv.getTrackerStats().getModelTimes().getNamedList(detail, hist, values));
coreSummary.add("Docs/Tx", srv.getTrackerStats().getTxDocs().getNamedList(detail, hist, values));
// Model
Map<String, Set<String>> modelErrors = srv.getModelErrors();
if (modelErrors.size() > 0)
{
NamedList<Object> errorList = new SimpleOrderedMap<>();
for (Map.Entry<String, Set<String>> modelNameToErrors : modelErrors.entrySet())
{
errorList.add(modelNameToErrors.getKey(), modelNameToErrors.getValue());
}
coreSummary.add("Model changes are not compatible with the existing data model and have not been applied", errorList);
}
report.add(cname, coreSummary);
}
static void addMasterOrStandaloneCoreSummary(TrackerRegistry trackerRegistry, String cname, boolean detail, boolean hist, boolean values,
InformationServer srv, NamedList<Object> report) throws IOException
{
NamedList<Object> coreSummary = new SimpleOrderedMap<>();
coreSummary.addAll((SimpleOrderedMap<Object>) srv.getCoreStats());
MetadataTracker metaTrkr = trackerRegistry.getTrackerForCore(cname, MetadataTracker.class);
TrackerState metadataTrkrState = metaTrkr.getTrackerState();
long lastIndexTxCommitTime = metadataTrkrState.getLastIndexedTxCommitTime();
long lastIndexedTxId = metadataTrkrState.getLastIndexedTxId();
long lastTxCommitTimeOnServer = metadataTrkrState.getLastTxCommitTimeOnServer();
long lastTxIdOnServer = metadataTrkrState.getLastTxIdOnServer();
Date lastIndexTxCommitDate = new Date(lastIndexTxCommitTime);
Date lastTxOnServerDate = new Date(lastTxCommitTimeOnServer);
long transactionsToDo = lastTxIdOnServer - lastIndexedTxId;
if (transactionsToDo < 0)
{
transactionsToDo = 0;
}
AclTracker aclTrkr = trackerRegistry.getTrackerForCore(cname, AclTracker.class);
TrackerState aclTrkrState = aclTrkr.getTrackerState();
long lastIndexChangeSetCommitTime = aclTrkrState.getLastIndexedChangeSetCommitTime();
long lastIndexedChangeSetId = aclTrkrState.getLastIndexedChangeSetId();
long lastChangeSetCommitTimeOnServer = aclTrkrState.getLastChangeSetCommitTimeOnServer();
long lastChangeSetIdOnServer = aclTrkrState.getLastChangeSetIdOnServer();
Date lastIndexChangeSetCommitDate = new Date(lastIndexChangeSetCommitTime);
Date lastChangeSetOnServerDate = new Date(lastChangeSetCommitTimeOnServer);
long changeSetsToDo = lastChangeSetIdOnServer - lastIndexedChangeSetId;
if (changeSetsToDo < 0)
{
changeSetsToDo = 0;
}
long nodesToDo = 0;
long remainingTxTimeMillis = 0;
if (transactionsToDo > 0)
{
// We now use the elapsed time as seen by the single thread farming out metadata indexing
double meanDocsPerTx = srv.getTrackerStats().getMeanDocsPerTx();
double meanNodeElaspedIndexTime = srv.getTrackerStats().getMeanNodeElapsedIndexTime();
nodesToDo = (long)(transactionsToDo * meanDocsPerTx);
remainingTxTimeMillis = (long) (nodesToDo * meanNodeElaspedIndexTime);
}
Date now = new Date();
Date end = new Date(now.getTime() + remainingTxTimeMillis);
Duration remainingTx = new Duration(now, end);
long remainingChangeSetTimeMillis = 0;
if (changeSetsToDo > 0)
{
// We now use the elapsed time as seen by the single thread farming out alc indexing
double meanAclsPerChangeSet = srv.getTrackerStats().getMeanAclsPerChangeSet();
double meanAclElapsedIndexTime = srv.getTrackerStats().getMeanAclElapsedIndexTime();
remainingChangeSetTimeMillis = (long) (changeSetsToDo * meanAclsPerChangeSet * meanAclElapsedIndexTime);
}
now = new Date();
end = new Date(now.getTime() + remainingChangeSetTimeMillis);
Duration remainingChangeSet = new Duration(now, end);
NamedList<Object> ftsSummary = new SimpleOrderedMap<>();
long remainingContentTimeMillis = 0;
srv.addFTSStatusCounts(ftsSummary);
long cleanCount =
ofNullable(ftsSummary.get("Node count with FTSStatus Clean"))
.map(Number.class::cast)
.map(Number::longValue)
.orElse(0L);
long dirtyCount =
ofNullable(ftsSummary.get("Node count with FTSStatus Dirty"))
.map(Number.class::cast)
.map(Number::longValue)
.orElse(0L);
long newCount =
ofNullable(ftsSummary.get("Node count with FTSStatus New"))
.map(Number.class::cast)
.map(Number::longValue)
.orElse(0L);
long nodesInIndex =
ofNullable(coreSummary.get("Alfresco Nodes in Index"))
.map(Number.class::cast)
.map(Number::longValue)
.orElse(0L);
long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount + newCount)/nodesInIndex : 0;
if (dirtyCount + newCount + contentYetToSee > 0)
{
// We now use the elapsed time as seen by the single thread farming out alc indexing
double meanContentElapsedIndexTime = srv.getTrackerStats().getMeanContentElapsedIndexTime();
remainingContentTimeMillis = (long) ((dirtyCount + newCount + contentYetToSee) * meanContentElapsedIndexTime);
}
now = new Date();
end = new Date(now.getTime() + remainingContentTimeMillis);
Duration remainingContent = new Duration(now, end);
coreSummary.add("FTS",ftsSummary);
Duration txLag = new Duration(lastIndexTxCommitDate, lastTxOnServerDate);
if (lastIndexTxCommitDate.compareTo(lastTxOnServerDate) > 0)
{
txLag = new Duration();
}
long txLagSeconds = (lastTxCommitTimeOnServer - lastIndexTxCommitTime) / 1000;
if (txLagSeconds < 0)
{
txLagSeconds = 0;
}
Duration changeSetLag = new Duration(lastIndexChangeSetCommitDate, lastChangeSetOnServerDate);
if (lastIndexChangeSetCommitDate.compareTo(lastChangeSetOnServerDate) > 0)
{
changeSetLag = new Duration();
}
long changeSetLagSeconds = (lastChangeSetCommitTimeOnServer - lastIndexChangeSetCommitTime) / 1000;
if (txLagSeconds < 0)
{
txLagSeconds = 0;
}
ContentTracker contentTrkr = trackerRegistry.getTrackerForCore(cname, ContentTracker.class);
TrackerState contentTrkrState = contentTrkr.getTrackerState();
// Leave ModelTracker out of this check, because it is common
boolean aTrackerIsRunning = aclTrkrState.isRunning() || metadataTrkrState.isRunning()
|| contentTrkrState.isRunning();
coreSummary.add("Active", aTrackerIsRunning);
ModelTracker modelTrkr = trackerRegistry.getModelTracker();
TrackerState modelTrkrState = modelTrkr.getTrackerState();
coreSummary.add("ModelTracker Active", modelTrkrState.isRunning());
coreSummary.add("ContentTracker Active", contentTrkrState.isRunning());
coreSummary.add("MetadataTracker Active", metadataTrkrState.isRunning());
coreSummary.add("AclTracker Active", aclTrkrState.isRunning());
// TX
coreSummary.add("Last Index TX Commit Time", lastIndexTxCommitTime);
coreSummary.add("Last Index TX Commit Date", lastIndexTxCommitDate);
coreSummary.add("TX Lag", txLagSeconds + " s");
coreSummary.add("TX Duration", txLag.toString());
coreSummary.add("Timestamp for last TX on server", lastTxCommitTimeOnServer);
coreSummary.add("Date for last TX on server", lastTxOnServerDate);
coreSummary.add("Id for last TX on server", lastTxIdOnServer);
coreSummary.add("Id for last TX in index", lastIndexedTxId);
coreSummary.add("Approx transactions remaining", transactionsToDo);
coreSummary.add("Approx transaction indexing time remaining", remainingTx.largestComponentformattedString());
// Change set
coreSummary.add("Last Index Change Set Commit Time", lastIndexChangeSetCommitTime);
coreSummary.add("Last Index Change Set Commit Date", lastIndexChangeSetCommitDate);
coreSummary.add("Change Set Lag", changeSetLagSeconds + " s");
coreSummary.add("Change Set Duration", changeSetLag.toString());
coreSummary.add("Timestamp for last Change Set on server", lastChangeSetCommitTimeOnServer);
coreSummary.add("Date for last Change Set on server", lastChangeSetOnServerDate);
coreSummary.add("Id for last Change Set on server", lastChangeSetIdOnServer);
coreSummary.add("Id for last Change Set in index", lastIndexedChangeSetId);
coreSummary.add("Approx change sets remaining", changeSetsToDo);
coreSummary.add("Approx change set indexing time remaining",
remainingChangeSet.largestComponentformattedString());
coreSummary.add("Approx content indexing time remaining",
remainingContent.largestComponentformattedString());
// Stats
coreSummary.add("Model sync times (ms)",
srv.getTrackerStats().getModelTimes().getNamedList(detail, hist, values));
coreSummary.add("Acl index time (ms)",
srv.getTrackerStats().getAclTimes().getNamedList(detail, hist, values));
coreSummary.add("Node index time (ms)",
srv.getTrackerStats().getNodeTimes().getNamedList(detail, hist, values));
coreSummary.add("Docs/Tx", srv.getTrackerStats().getTxDocs().getNamedList(detail, hist, values));
coreSummary.add("Doc Transformation time (ms)", srv.getTrackerStats().getDocTransformationTimes()
.getNamedList(detail, hist, values));
// Model
Map<String, Set<String>> modelErrors = srv.getModelErrors();
if (modelErrors.size() > 0)
{
NamedList<Object> errorList = new SimpleOrderedMap<>();
for (Map.Entry<String, Set<String>> modelNameToErrors : modelErrors.entrySet())
{
errorList.add(modelNameToErrors.getKey(), modelNameToErrors.getValue());
}
coreSummary.add("Model changes are not compatible with the existing data model and have not been applied",
errorList);
}
report.add(cname, coreSummary);
}
}

6
search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java
View File

@@ -35,7 +35,7 @@ import org.alfresco.solr.tracker.CommitTracker;
import org.alfresco.solr.tracker.ContentTracker;
import org.alfresco.solr.tracker.MetadataTracker;
import org.alfresco.solr.tracker.ModelTracker;
import org.alfresco.solr.tracker.SlaveNodeStatePublisher;
import org.alfresco.solr.tracker.SlaveCoreStatePublisher;
import org.alfresco.solr.tracker.SolrTrackerScheduler;
import org.alfresco.solr.tracker.Tracker;
import org.alfresco.solr.tracker.TrackerRegistry;
@@ -182,7 +182,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener
{
LOGGER.info("SearchServices Core Trackers have been explicitly disabled on core \"{}\" through \"enable.alfresco.tracking\" configuration property.", core.getName());
SlaveNodeStatePublisher statePublisher = new SlaveNodeStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer);
SlaveCoreStatePublisher statePublisher = new SlaveCoreStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer);
trackerRegistry.register(core.getName(), statePublisher);
scheduler.schedule(statePublisher, core.getName(), coreProperties);
trackers.add(statePublisher);
@@ -197,7 +197,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener
{
LOGGER.info("SearchServices Core Trackers have been disabled on core \"{}\" because it is a slave core.", core.getName());
SlaveNodeStatePublisher statePublisher = new SlaveNodeStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer);
SlaveCoreStatePublisher statePublisher = new SlaveCoreStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer);
trackerRegistry.register(core.getName(), statePublisher);
scheduler.schedule(statePublisher, core.getName(), coreProperties);
trackers.add(statePublisher);

33
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/NodeStatePublisher.java → search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java
View File

@@ -41,6 +41,7 @@ import org.alfresco.service.namespace.QName;
import org.alfresco.solr.AlfrescoCoreAdminHandler;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.InformationServer;
import org.alfresco.solr.NodeReport;
import org.alfresco.solr.TrackerState;
import org.alfresco.solr.client.SOLRAPIClient;
import org.apache.commons.lang3.StringUtils;
@@ -59,7 +60,7 @@ import java.util.Properties;
* @since 1.5
* @see <a href="https://issues.alfresco.com/jira/browse/SEARCH-1752">SEARCH-1752</a>
*/
public abstract class NodeStatePublisher extends AbstractTracker
public abstract class CoreStatePublisher extends AbstractTracker
{
DocRouter docRouter;
private final boolean isMaster;
@@ -70,7 +71,7 @@ public abstract class NodeStatePublisher extends AbstractTracker
/** The property to use for determining the shard. */
protected Optional<QName> shardProperty = Optional.empty();
NodeStatePublisher(
CoreStatePublisher(
boolean isMaster,
Properties p,
SOLRAPIClient client,
@@ -88,12 +89,28 @@ public abstract class NodeStatePublisher extends AbstractTracker
docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod));
}
NodeStatePublisher(Type type)
CoreStatePublisher(Type type)
{
super(type);
this.isMaster = false;
}
/**
* Returns information about the {@link org.alfresco.solr.client.Node} associated with the given dbid.
*
* @param dbid the node identifier.
* @return the {@link org.alfresco.solr.client.Node} associated with the given dbid.
*/
public NodeReport checkNode(Long dbid)
{
NodeReport nodeReport = new NodeReport();
nodeReport.setDbid(dbid);
this.infoSrv.addCommonNodeReportInfo(nodeReport);
return nodeReport;
}
private void firstUpdateShardProperty()
{
shardKey.ifPresent( shardKeyName -> {
@@ -222,4 +239,14 @@ public abstract class NodeStatePublisher extends AbstractTracker
{
return this.docRouter;
}
/**
* Returns true if the hosting core is master or standalone.
*
* @return true if the hosting core is master or standalone.
*/
public boolean isOnMasterOrStandalone()
{
return isMaster;
}
}

34
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java
View File

@@ -49,7 +49,7 @@ import org.slf4j.LoggerFactory;
* This tracks two things: transactions and metadata nodes
* @author Ahmed Owian
*/
public class MetadataTracker extends NodeStatePublisher implements Tracker
public class MetadataTracker extends CoreStatePublisher implements Tracker
{
protected final static Logger log = LoggerFactory.getLogger(MetadataTracker.class);
private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 100;
@@ -888,24 +888,18 @@ public class MetadataTracker extends NodeStatePublisher implements Tracker
}
}
@Override
public NodeReport checkNode(Long dbid)
{
NodeReport nodeReport = new NodeReport();
nodeReport.setDbid(dbid);
NodeReport nodeReport = super.checkNode(dbid);
// In DB
GetNodesParameters parameters = new GetNodesParameters();
parameters.setFromNodeId(dbid);
parameters.setToNodeId(dbid);
List<Node> dbnodes;
try
{
dbnodes = client.getNodes(parameters, 1);
List<Node> dbnodes = client.getNodes(parameters, 1);
if (dbnodes.size() == 1)
{
Node dbnode = dbnodes.get(0);
@@ -915,41 +909,31 @@ public class MetadataTracker extends NodeStatePublisher implements Tracker
else
{
nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN);
nodeReport.setDbTx(-1l);
nodeReport.setDbTx(-1L);
}
}
catch (IOException e)
{
nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN);
nodeReport.setDbTx(-2l);
nodeReport.setDbTx(-2L);
}
catch (JSONException e)
{
nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN);
nodeReport.setDbTx(-3l);
nodeReport.setDbTx(-3L);
}
catch (AuthenticationException e1)
{
nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN);
nodeReport.setDbTx(-4l);
nodeReport.setDbTx(-4L);
}
this.infoSrv.addCommonNodeReportInfo(nodeReport);
return nodeReport;
}
public NodeReport checkNode(Node node)
{
NodeReport nodeReport = new NodeReport();
nodeReport.setDbid(node.getId());
nodeReport.setDbNodeStatus(node.getStatus());
nodeReport.setDbTx(node.getTxnId());
this.infoSrv.addCommonNodeReportInfo(nodeReport);
return nodeReport;
return checkNode(node.getId());
}
public List<Node> getFullNodesForDbTransaction(Long txid)

14
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveNodeStatePublisher.java → search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java
View File

@@ -14,8 +14,8 @@ import java.util.Properties;
/**
* Despite belonging to the Tracker ecosystem, this component is actually a publisher, which periodically informs
* Alfresco about the state of the hosting slave node.
* As the name suggests, this worker is scheduled only when the hosting node acts as a slave.
* Alfresco about the state of the hosting slave core.
* As the name suggests, this worker is scheduled only when the owning core acts as a slave.
* It allows Solr's master/slave setup to be used with dynamic shard registration.
*
* In this scenario the slave is polling a "tracking" Solr node. The tracker below calls
@@ -27,9 +27,9 @@ import java.util.Properties;
* @author Andrea Gazzarini
* @since 1.5
*/
public class SlaveNodeStatePublisher extends NodeStatePublisher
public class SlaveCoreStatePublisher extends CoreStatePublisher
{
public SlaveNodeStatePublisher(
public SlaveCoreStatePublisher(
boolean isMaster,
Properties coreProperties,
SOLRAPIClient repositoryClient,
@@ -61,6 +61,12 @@ public class SlaveNodeStatePublisher extends NodeStatePublisher
// Do nothing here
}
@Override
public boolean isOnMasterOrStandalone()
{
return false;
}
@Override
public boolean hasMaintenance()
{

56
search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java Normal file
View File

@@ -0,0 +1,56 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.utils;
import java.util.Collection;
import java.util.Collections;
public abstract class Utils
{
/**
* Returns the same input collection if that is not null, otherwise a new empty collection.
* Provides a safe way for iterating over a returned collection (which could be null).
*
* @param values the collection.
* @param <T> the collection type.
* @return the same input collection if that is not null, otherwise a new empty collection.
*/
public static <T> Collection<T> notNullOrEmpty(Collection<T> values)
{
return values != null ? values : Collections.emptyList();
}
/**
* Converts the given input in an Integer, otherwise it returns null.
*
* @param value the numeric string.
* @return the corresponding Integer or null in case the input is NaN.
*/
public static Integer toIntOrNull(String value)
{
try
{
return Integer.valueOf(value);
}
catch(NumberFormatException nfe)
{
return null;
}
}
}

187
search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java
View File

@@ -20,15 +20,23 @@ package org.alfresco.solr;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.ALFRESCO_CORE_NAME;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARCHIVE_CORE_NAME;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARG_TXID;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.STORE_REF_MAP;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.VERSION_CORE_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.spy;
@@ -42,13 +50,18 @@ import java.util.stream.Collectors;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.solr.adapters.IOpenBitSet;
import org.alfresco.solr.tracker.AclTracker;
import org.alfresco.solr.tracker.DocRouter;
import org.alfresco.solr.tracker.IndexHealthReport;
import org.alfresco.solr.tracker.MetadataTracker;
import org.alfresco.solr.tracker.PropertyRouter;
import org.alfresco.solr.tracker.SlaveCoreStatePublisher;
import org.alfresco.solr.tracker.TrackerRegistry;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.junit.Before;
@@ -105,6 +118,169 @@ public class AlfrescoCoreAdminHandlerIT
when(req.getParams()).thenReturn(params);
}
@Test
public void extractShardsWithEmptyParameter_shouldReturnAnEmptyList()
{
assertTrue(alfrescoCoreAdminHandler.extractShards("", Integer.MAX_VALUE).isEmpty());
}
@Test
public void extractShardsWithNullParameter_shouldReturnAnEmptyList()
{
assertTrue(alfrescoCoreAdminHandler.extractShards(null, Integer.MAX_VALUE).isEmpty());
}
@Test
public void extractShardsWithOneInvalidShard_shouldReturnAnEmptyList()
{
assertTrue(alfrescoCoreAdminHandler.extractShards("This is an invalid shard id", Integer.MAX_VALUE).isEmpty());
}
@Test
public void extractShardsWithOneShards_shouldReturnSingletonList()
{
assertEquals(singletonList(1), alfrescoCoreAdminHandler.extractShards("1", Integer.MAX_VALUE));
}
@Test
public void extractShardsWithSeveralValidShards_shouldReturnAllOfThemInTheList()
{
assertEquals(asList(1,5,6,11,23), alfrescoCoreAdminHandler.extractShards("1,5,6,11,23", Integer.MAX_VALUE));
}
@Test
public void extractShardsWithSeveralValidShards_shouldReturnOnlyValidIdentifiers()
{
assertEquals(asList(1,5,6,11,23), alfrescoCoreAdminHandler.extractShards("1,5,A,6,xyz,11,BB,23,o01z", Integer.MAX_VALUE));
}
@Test
public void extractShardsWithSeveralValidShardsAndLimit_shouldConsiderOnlyShardsLesserThanLimit()
{
assertEquals(asList(1,5,6,11,12), alfrescoCoreAdminHandler.extractShards("1,5,6,11,23,25,99,223,12", 23));
}
@Test
public void hasAlfrescoCoreWhenInputIsNull_shouldReturnFalse()
{
assertFalse(alfrescoCoreAdminHandler.hasAlfrescoCore(null));
}
@Test
public void hasAlfrescoCoreWhenWeHaveNoCore_shouldReturnFalse()
{
assertFalse(alfrescoCoreAdminHandler.hasAlfrescoCore(emptyList()));
}
@Test
public void hasAlfrescoCoreWhenDoesntHaveAnyTracker_shouldReturnFalse()
{
assertFalse(alfrescoCoreAdminHandler.hasAlfrescoCore(emptyList()));
}
@Test
public void hasAlfrescoCoreWithRegisteredTrackers_shouldReturnTrue()
{
when(trackerRegistry.hasTrackersForCore("CoreD")).thenReturn(true);
assertTrue(alfrescoCoreAdminHandler.hasAlfrescoCore(asList(dummyCore("CoreA"), dummyCore("CoreB"), dummyCore("CoreC"), dummyCore("CoreD"))));
}
@Test
public void trackerRegistryHasNoCoreNames_itShouldReturnAnEmptyList()
{
assertTrue(alfrescoCoreAdminHandler.coreNames().isEmpty());
}
@Test
public void coreDetectedAsMasterOrStandalone()
{
MetadataTracker coreStatePublisher = mock(MetadataTracker.class);
when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class)))
.thenReturn(coreStatePublisher);
assertTrue(alfrescoCoreAdminHandler.isMasterOrStandalone("ThisIsTheCoreName"));
}
@Test
public void coreDetectedAsSlave()
{
when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))).thenReturn(null);
assertFalse(alfrescoCoreAdminHandler.isMasterOrStandalone("ThisIsTheCoreName"));
}
@Test
public void coreIsMaster_thenCoreStatePublisherInstanceCorrespondsToMetadataTracker()
{
MetadataTracker coreStatePublisher = mock(MetadataTracker.class);
when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class)))
.thenReturn(coreStatePublisher);
assertSame(coreStatePublisher, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName"));
}
@Test
public void coreIsSlave_thenCoreStatePublisherInstanceCorrespondsToSlaveCoreStatePublisher()
{
SlaveCoreStatePublisher coreStatePublisher = mock(SlaveCoreStatePublisher.class);
when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))).thenReturn(null);
when(trackerRegistry.getTrackerForCore(anyString(), eq(SlaveCoreStatePublisher.class))).thenReturn(coreStatePublisher);
assertSame(coreStatePublisher, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName"));
}
@Test
public void coreIsSlave_thenDocRouterIsNull()
{
String coreName = "aCore";
when(trackerRegistry.getTrackerForCore(eq(coreName), eq(MetadataTracker.class))).thenReturn(null);
assertNull(alfrescoCoreAdminHandler.getDocRouter("aCore"));
}
@Test
public void coreIsMaster_thenDocRouterIsProperlyReturned()
{
DocRouter expectedRouter = new PropertyRouter("someProperty_.{1,35}");
MetadataTracker coreStatePublisher = mock(MetadataTracker.class);
when(coreStatePublisher.getDocRouter()).thenReturn(expectedRouter);
when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))).thenReturn(coreStatePublisher);
assertSame(expectedRouter, alfrescoCoreAdminHandler.getDocRouter("aCore"));
}
@Test
public void targetCoreNameCanBeSpecifiedInSeveralWays()
{
String coreName = "ThisIsTheCoreName";
ModifiableSolrParams params = new ModifiableSolrParams();
assertNull(alfrescoCoreAdminHandler.coreName(params));
params.set(CoreAdminParams.CORE, coreName);
assertEquals(coreName, alfrescoCoreAdminHandler.coreName(params));
params.remove(CoreAdminParams.CORE);
assertNull(alfrescoCoreAdminHandler.coreName(params));
params.set("coreName", coreName);
assertEquals(coreName, alfrescoCoreAdminHandler.coreName(params));
assertEquals(coreName, alfrescoCoreAdminHandler.coreName(params));
}
private SolrCore dummyCore(String name)
{
SolrCore core = mock(SolrCore.class);
when(core.getName()).thenReturn(name);
return core;
}
/** Check that a transaction report can be generated. */
@Test
public void handleCustomActionTXReportSuccess() throws Exception
@@ -143,8 +319,6 @@ public class AlfrescoCoreAdminHandlerIT
public void handleCustomActionTXReportMissingTXId()
{
when(params.get(CoreAdminParams.ACTION)).thenReturn(TXREPORT);
when(params.get(ARG_TXID)).thenReturn(null);
alfrescoCoreAdminHandler.handleCustomAction(req, rsp);
verify(rsp, never()).add(anyString(), any());
@@ -156,11 +330,8 @@ public class AlfrescoCoreAdminHandlerIT
{
when(params.get(CoreAdminParams.ACTION)).thenReturn(TXREPORT);
when(params.get(CoreAdminParams.CORE)).thenReturn(null);
when(params.get(ARG_TXID)).thenReturn(TX_ID);
alfrescoCoreAdminHandler.handleCustomAction(req, rsp);
verify(rsp, never()).add(anyString(), any());
}
/** Check that when an unknown action is provided we don't generate a report. */
@@ -212,10 +383,9 @@ public class AlfrescoCoreAdminHandlerIT
public void coreNamesAreTrimmed_oneCoreNameAtTime() {
AlfrescoCoreAdminHandler spy = spy(new AlfrescoCoreAdminHandler() {
@Override
protected boolean newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp)
protected void newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp)
{
// Do nothing here otherwise we cannot spy it
return true;
}
});
@@ -238,10 +408,9 @@ public class AlfrescoCoreAdminHandlerIT
public void validAndInvalidCoreNames() {
AlfrescoCoreAdminHandler spy = spy(new AlfrescoCoreAdminHandler() {
@Override
protected boolean newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp)
protected void newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp)
{
// Do nothing here otherwise we cannot spy it
return true;
}
});