From d9c1a153fb59daa1c5d5489c0e435ac65816bdce Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 6 Dec 2019 15:10:10 +0100 Subject: [PATCH 1/4] [SEARCH-1994] modified path computation in order to work correctly on windows. --- .../solr/handler/AlfrescoIndexFetcher.java | 141 +++++++++--------- 1 file changed, 72 insertions(+), 69 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index ee1666778..3723b22e7 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -34,39 +34,9 @@ */ package org.alfresco.solr.handler; -import static java.util.List.of; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.ALIAS; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CHECKSUM; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_CONTENT_STORE_FILES; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_DETAILS; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE_LIST; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_INDEX_VERSION; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMMAND; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMPRESSION; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILES; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILE_SHORT; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILES; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILE_LIST; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_VERSION; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.EXTERNAL; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE_STREAM; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FileInfo; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.GENERATION; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.INTERNAL; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.MASTER_URL; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.NO_INDEX_REPLICATION_REQUIRED; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.OFFSET; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.SIZE; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILE; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILES; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.getCheckSum; -import static org.apache.solr.common.params.CommonParams.JAVABIN; -import static org.apache.solr.common.params.CommonParams.NAME; - import com.google.common.base.Strings; import com.google.common.collect.Lists; +import jdk.internal.jline.internal.Log; import org.alfresco.solr.content.SolrContentStore; import org.apache.http.client.HttpClient; import org.apache.lucene.codecs.CodecUtil; @@ -152,6 +122,37 @@ import java.util.zip.Adler32; import java.util.zip.Checksum; import java.util.zip.InflaterInputStream; +import static java.util.List.of; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.ALIAS; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CHECKSUM; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_CONTENT_STORE_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_DETAILS; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE_LIST; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_INDEX_VERSION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMMAND; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMPRESSION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILE_SHORT; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILE_LIST; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_VERSION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.EXTERNAL; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE_STREAM; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FileInfo; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.GENERATION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.INTERNAL; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.MASTER_URL; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.NO_INDEX_REPLICATION_REQUIRED; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.OFFSET; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.SIZE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.getCheckSum; +import static org.apache.solr.common.params.CommonParams.JAVABIN; +import static org.apache.solr.common.params.CommonParams.NAME; + /** *

Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the * master.

@@ -776,30 +777,37 @@ class AlfrescoIndexFetcher } } - if (contentStoreReplicationNeeded) - { + try { - if (contentStoreFilesToDownload != null) + if (contentStoreReplicationNeeded) { - bytesDownloaded += downloadContentStoreFiles(contentStore.getRootLocation()); + + if (contentStoreFilesToDownload != null) + { + bytesDownloaded += downloadContentStoreFiles(contentStore.getRootLocation()); + } + + if (contentStoreFilesToDelete != null) + { + deleteContentStoreFiles(contentStore.getRootLocation(), contentStoreFilesToDelete); + } + + if (fullContentStoreReplication) + { + cleanUpContentStore(contentStore.getRootLocation()); + } + + contentStore.setLastCommittedVersion(masterContentStoreVersion); + LOG.info("content store has been updated to version: {}", masterContentStoreVersion); } - if (contentStoreFilesToDelete != null) - { - deleteContentStoreFiles(contentStore.getRootLocation(), contentStoreFilesToDelete); - } - - if (fullContentStoreReplication) - { - cleanUpContentStore(contentStore.getRootLocation()); - } - - contentStore.setLastCommittedVersion(masterContentStoreVersion); + } catch (Exception e) { + LOG.error("impossible to complete content store replication {}", e); } final long timeTakenSeconds = getReplicationTimeElapsed(); final Long bytesDownloadedPerSecond = (timeTakenSeconds != 0 ? bytesDownloaded / timeTakenSeconds : - null); + null); LOG.info("Total time taken for download (fullCopy={},bytesDownloaded={}) : {} secs ({} bytes/sec)", isFullCopyNeeded, bytesDownloaded, timeTakenSeconds, bytesDownloadedPerSecond); @@ -1683,37 +1691,32 @@ class AlfrescoIndexFetcher * @param contentStorePath * @throws IOException */ - private void copyTmpContentStoreToContentStore(File tmpContentStoreDir, String contentStorePath) throws IOException + private void copyTmpContentStoreToContentStore(File tmpContentStoreDir, String contentStorePath) throws Exception { String tmpContentStorePath = tmpContentStoreDir.getPath(); - try - { - Files.walk(tmpContentStoreDir.toPath()).forEach(p -> { - File tmpFile = new File(p.toUri()); - if (!tmpFile.isDirectory()) + Files.walk(tmpContentStoreDir.toPath()).forEach(p -> { + File tmpFile = new File(p.toUri()); + if (!tmpFile.isDirectory()) + { + File csFile = new File(p.toString().replace(tmpContentStorePath, contentStorePath)); + try { - File csFile = new File(p.toString().replaceFirst(tmpContentStorePath, contentStorePath)); - try - { - Files.createDirectories(Paths.get(csFile.getParent())); - tmpFile.renameTo(csFile); - } - catch (IOException e) - { - LOG.error("impossible to copy {}", csFile.toString()); + Files.createDirectories(Paths.get(csFile.getParent())); + if (!tmpFile.renameTo(csFile)){ + throw new RuntimeException("Failed while moving content store file to " + csFile.getAbsolutePath()); } } - }); - } - catch (IOException e) - { - LOG.error("impossible tmp content store"); - throw e; - } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + }); } + /** * Deletes the files in filesToDelete list from contentStore * @param contentStorePath From b6f038c09c3af29223bc4d5ab686f7b974fe7632 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 6 Dec 2019 16:35:51 +0100 Subject: [PATCH 2/4] [SEARCH-1994] fix compilation error --- .../java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java | 1 - 1 file changed, 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 3723b22e7..477a38dec 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -36,7 +36,6 @@ package org.alfresco.solr.handler; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import jdk.internal.jline.internal.Log; import org.alfresco.solr.content.SolrContentStore; import org.apache.http.client.HttpClient; import org.apache.lucene.codecs.CodecUtil; From 80e31a85753ba6d7f6513869f899d37be1f43ee6 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Mon, 9 Dec 2019 14:17:48 +0100 Subject: [PATCH 3/4] [SEARCH-1994] some fixes to make contentstore replication working on windows --- .../alfresco/solr/handler/AlfrescoIndexFetcher.java | 11 +++++------ .../solr/handler/ContentStoreReplicationIT.java | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 477a38dec..81e40fef3 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -1703,9 +1703,7 @@ class AlfrescoIndexFetcher try { Files.createDirectories(Paths.get(csFile.getParent())); - if (!tmpFile.renameTo(csFile)){ - throw new RuntimeException("Failed while moving content store file to " + csFile.getAbsolutePath()); - } + Files.copy(tmpFile.toPath(), csFile.toPath()); } catch (IOException e) { @@ -1735,7 +1733,7 @@ class AlfrescoIndexFetcher * Deletes from contentstore all the files that has not been updated. * @param contentStorePath */ - private void cleanUpContentStore(String contentStorePath) + private void cleanUpContentStore(String contentStorePath) throws Exception { AtomicInteger fileDeleted = new AtomicInteger(); Set fileNames = contentStoreFilesToDownload.stream().map(e -> (String) e.get(NAME)) @@ -1744,7 +1742,7 @@ class AlfrescoIndexFetcher { Files.walk(Paths.get(contentStorePath)).forEach(p -> { File f = new File(p.toUri()); - if (!f.isDirectory() && !fileNames.contains(p.toString().replaceFirst(contentStorePath, ""))) + if (!f.isDirectory() && !fileNames.contains(p.toString().replace(contentStorePath, ""))) { try { @@ -1758,9 +1756,10 @@ class AlfrescoIndexFetcher } }); } - catch (IOException e) + catch (Exception e) { LOG.error("Impossible to delete unnecessary files. Content store may contains unused contents"); + throw(e); } LOG.info("deleted {} unnecessary files from content store", fileDeleted); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java index 8fb8eeb58..678afd5ad 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java @@ -141,8 +141,8 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT { master.stop(); slave.stop(); - FileUtils.forceDelete(new File(masterSolrHome.getParent().toUri())); - FileUtils.forceDelete(new File(slaveSolrHome.getParent().toUri())); + FileUtils.deleteQuietly(new File(masterSolrHome.getParent().toUri())); + FileUtils.deleteQuietly(new File(slaveSolrHome.getParent().toUri())); SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); From f908b11d49ff7ee9f3afd1ca2ae1ed208c400f2e Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Mon, 9 Dec 2019 14:31:46 +0100 Subject: [PATCH 4/4] [SEARCH-1994] replace existend file when copy tmp contentstore files --- .../java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 81e40fef3..66d676867 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -1703,7 +1703,7 @@ class AlfrescoIndexFetcher try { Files.createDirectories(Paths.get(csFile.getParent())); - Files.copy(tmpFile.toPath(), csFile.toPath()); + Files.copy(tmpFile.toPath(), csFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) {