inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2026-09-16 18:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'fix/SEARCH-1994' into 'master'

Browse Source 
Fix/search 1994

See merge request search_discovery/insightengine!255
This commit is contained in:
Elia Porciani
2019-12-10 14:06:37 +00:00
parent 5dc0718f9f f908b11d49
commit dc9840ac4d
2 changed files with 76 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java
+74 -73
View File
@@ -34,37 +34,6 @@
*/
package org.alfresco.solr.handler;
import static java.util.List.of;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.ALIAS;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CHECKSUM;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_CONTENT_STORE_FILES;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_DETAILS;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE_LIST;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_INDEX_VERSION;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMMAND;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMPRESSION;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILES;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILE_SHORT;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILES;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILE_LIST;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_VERSION;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.EXTERNAL;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE_STREAM;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FileInfo;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.GENERATION;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.INTERNAL;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.MASTER_URL;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.NO_INDEX_REPLICATION_REQUIRED;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.OFFSET;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.SIZE;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILE;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILES;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.getCheckSum;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.NAME;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import org.alfresco.solr.content.SolrContentStore;
@@ -152,6 +121,37 @@ import java.util.zip.Adler32;
import java.util.zip.Checksum;
import java.util.zip.InflaterInputStream;
import static java.util.List.of;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.ALIAS;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CHECKSUM;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_CONTENT_STORE_FILES;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_DETAILS;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE_LIST;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_INDEX_VERSION;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMMAND;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMPRESSION;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILES;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILE_SHORT;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILES;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILE_LIST;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_VERSION;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.EXTERNAL;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE_STREAM;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FileInfo;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.GENERATION;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.INTERNAL;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.MASTER_URL;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.NO_INDEX_REPLICATION_REQUIRED;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.OFFSET;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.SIZE;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILE;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILES;
import static org.alfresco.solr.handler.AlfrescoReplicationHandler.getCheckSum;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.NAME;
/**
* <p> Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the
* master. </p>
@@ -776,30 +776,37 @@ class AlfrescoIndexFetcher
}
}
if (contentStoreReplicationNeeded)
{
try {
if (contentStoreFilesToDownload != null)
if (contentStoreReplicationNeeded)
{
bytesDownloaded += downloadContentStoreFiles(contentStore.getRootLocation());
if (contentStoreFilesToDownload != null)
{
bytesDownloaded += downloadContentStoreFiles(contentStore.getRootLocation());
}
if (contentStoreFilesToDelete != null)
{
deleteContentStoreFiles(contentStore.getRootLocation(), contentStoreFilesToDelete);
}
if (fullContentStoreReplication)
{
cleanUpContentStore(contentStore.getRootLocation());
}
contentStore.setLastCommittedVersion(masterContentStoreVersion);
LOG.info("content store has been updated to version: {}", masterContentStoreVersion);
}
if (contentStoreFilesToDelete != null)
{
deleteContentStoreFiles(contentStore.getRootLocation(), contentStoreFilesToDelete);
}
if (fullContentStoreReplication)
{
cleanUpContentStore(contentStore.getRootLocation());
}
contentStore.setLastCommittedVersion(masterContentStoreVersion);
} catch (Exception e) {
LOG.error("impossible to complete content store replication {}", e);
}
final long timeTakenSeconds = getReplicationTimeElapsed();
final Long bytesDownloadedPerSecond = (timeTakenSeconds != 0 ? bytesDownloaded / timeTakenSeconds :
null);
null);
LOG.info("Total time taken for download (fullCopy={},bytesDownloaded={}) : {} secs ({} bytes/sec)",
isFullCopyNeeded, bytesDownloaded, timeTakenSeconds, bytesDownloadedPerSecond);
@@ -1683,37 +1690,30 @@ class AlfrescoIndexFetcher
* @param contentStorePath
* @throws IOException
*/
private void copyTmpContentStoreToContentStore(File tmpContentStoreDir, String contentStorePath) throws IOException
private void copyTmpContentStoreToContentStore(File tmpContentStoreDir, String contentStorePath) throws Exception
{
String tmpContentStorePath = tmpContentStoreDir.getPath();
try
{
Files.walk(tmpContentStoreDir.toPath()).forEach(p -> {
File tmpFile = new File(p.toUri());
if (!tmpFile.isDirectory())
Files.walk(tmpContentStoreDir.toPath()).forEach(p -> {
File tmpFile = new File(p.toUri());
if (!tmpFile.isDirectory())
{
File csFile = new File(p.toString().replace(tmpContentStorePath, contentStorePath));
try
{
File csFile = new File(p.toString().replaceFirst(tmpContentStorePath, contentStorePath));
try
{
Files.createDirectories(Paths.get(csFile.getParent()));
tmpFile.renameTo(csFile);
}
catch (IOException e)
{
LOG.error("impossible to copy {}", csFile.toString());
}
Files.createDirectories(Paths.get(csFile.getParent()));
Files.copy(tmpFile.toPath(), csFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
}
});
}
catch (IOException e)
{
LOG.error("impossible tmp content store");
throw e;
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
});
}
/**
* Deletes the files in filesToDelete list from contentStore
* @param contentStorePath
@@ -1733,7 +1733,7 @@ class AlfrescoIndexFetcher
* Deletes from contentstore all the files that has not been updated.
* @param contentStorePath
*/
private void cleanUpContentStore(String contentStorePath)
private void cleanUpContentStore(String contentStorePath) throws Exception
{
AtomicInteger fileDeleted = new AtomicInteger();
Set<String> fileNames = contentStoreFilesToDownload.stream().map(e -> (String) e.get(NAME))
@@ -1742,7 +1742,7 @@ class AlfrescoIndexFetcher
{
Files.walk(Paths.get(contentStorePath)).forEach(p -> {
File f = new File(p.toUri());
if (!f.isDirectory() && !fileNames.contains(p.toString().replaceFirst(contentStorePath, "")))
if (!f.isDirectory() && !fileNames.contains(p.toString().replace(contentStorePath, "")))
{
try
{
@@ -1756,9 +1756,10 @@ class AlfrescoIndexFetcher
}
});
}
catch (IOException e)
catch (Exception e)
{
LOG.error("Impossible to delete unnecessary files. Content store may contains unused contents");
throw(e);
}
LOG.info("deleted {} unnecessary files from content store", fileDeleted);
search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java
+2 -2
View File
@@ -141,8 +141,8 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT
{
master.stop();
slave.stop();
FileUtils.forceDelete(new File(masterSolrHome.getParent().toUri()));
FileUtils.forceDelete(new File(slaveSolrHome.getParent().toUri()));
FileUtils.deleteQuietly(new File(masterSolrHome.getParent().toUri()));
FileUtils.deleteQuietly(new File(slaveSolrHome.getParent().toUri()));
SOLRAPIQueueClient.nodeMetaDataMap.clear();
SOLRAPIQueueClient.transactionQueue.clear();