From 980939c179e201ea6c81117e00305efced9375f6 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Wed, 25 Sep 2019 11:17:19 +0200 Subject: [PATCH 01/76] [SEARCH-1862] Added necessary classes for custom replication handler --- .../handler/AlfrescoReplicationHandler.java | 12 + .../alfresco/solr/handler/IndexFetcher.java | 2081 +++++++++++++++++ .../solr/handler/OldBackupDirectory.java | 53 + .../solr/handler/ReplicationHandler.java | 1934 +++++++++++++++ .../alfresco/solr/handler/SnapShooter.java | 288 +++ 5 files changed, 4368 insertions(+) create mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java create mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/IndexFetcher.java create mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java create mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java create mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java new file mode 100644 index 000000000..cdf308c4d --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java @@ -0,0 +1,12 @@ +package org.alfresco.solr.handler; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.invoke.MethodHandles; + + +public class AlfrescoReplicationHandler extends ReplicationHandler { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/IndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/IndexFetcher.java new file mode 100644 index 000000000..f32b7691d --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/IndexFetcher.java @@ -0,0 +1,2081 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.alfresco.solr.handler; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import org.alfresco.solr.content.ContentStoreCache; +import org.apache.http.client.HttpClient; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.FastInputStream; +import org.apache.solr.common.util.IOUtils; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SuppressForbidden; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.DirectoryFactory.DirContext; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.SnapShooter; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.CdcrUpdateLog; +import org.apache.solr.update.CommitUpdateCommand; +import org.apache.solr.update.UpdateLog; +import org.apache.solr.update.VersionInfo; +import org.apache.solr.util.DefaultSolrThreadFactory; +import org.apache.solr.util.FileUtils; +import org.apache.solr.util.PropertiesOutputStream; +import org.apache.solr.util.RTimer; +import org.apache.solr.util.RefCounted; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.lang.invoke.MethodHandles; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.zip.Adler32; +import java.util.zip.Checksum; +import java.util.zip.InflaterInputStream; + +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILE; +import static org.alfresco.solr.handler.ReplicationHandler.ALIAS; +import static org.alfresco.solr.handler.ReplicationHandler.CONTENT_STORE_FILES; +import static org.alfresco.solr.handler.ReplicationHandler.CHECKSUM; +import static org.alfresco.solr.handler.ReplicationHandler.CMD_CONTENT_STORE_FILES; +import static org.alfresco.solr.handler.ReplicationHandler.CMD_DETAILS; +import static org.alfresco.solr.handler.ReplicationHandler.CMD_GET_FILE; +import static org.alfresco.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST; +import static org.alfresco.solr.handler.ReplicationHandler.CMD_INDEX_VERSION; +import static org.alfresco.solr.handler.ReplicationHandler.COMMAND; +import static org.alfresco.solr.handler.ReplicationHandler.COMPRESSION; +import static org.alfresco.solr.handler.ReplicationHandler.CONF_FILES; +import static org.alfresco.solr.handler.ReplicationHandler.CONF_FILE_SHORT; +import static org.alfresco.solr.handler.ReplicationHandler.CONTENT_STORE_FILE_LIST; +import static org.alfresco.solr.handler.ReplicationHandler.EXTERNAL; +import static org.alfresco.solr.handler.ReplicationHandler.FILE; +import static org.alfresco.solr.handler.ReplicationHandler.FILE_STREAM; +import static org.alfresco.solr.handler.ReplicationHandler.FileInfo; +import static org.alfresco.solr.handler.ReplicationHandler.GENERATION; +import static org.alfresco.solr.handler.ReplicationHandler.INTERNAL; +import static org.alfresco.solr.handler.ReplicationHandler.MASTER_URL; +import static org.alfresco.solr.handler.ReplicationHandler.OFFSET; +import static org.alfresco.solr.handler.ReplicationHandler.PACKET_SZ; +import static org.alfresco.solr.handler.ReplicationHandler.SIZE; +import static org.alfresco.solr.handler.ReplicationHandler.TLOG_FILE; +import static org.alfresco.solr.handler.ReplicationHandler.TLOG_FILES; +import static org.apache.solr.common.params.CommonParams.JAVABIN; +import static org.apache.solr.common.params.CommonParams.NAME; + +/** + *

Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the + * master.

+ * + * + * @since solr 1.4 + * + * + * + */ +public class IndexFetcher { + private static final int _100K = 100000; + + public static final String INDEX_PROPERTIES = "index.properties"; + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final String masterUrl; + + final ReplicationHandler replicationHandler; + + private volatile Date replicationStartTimeStamp; + private RTimer replicationTimer; + + private final SolrCore solrCore; + + private volatile List> filesToDownload; + + private volatile List> confFilesToDownload; + + private volatile List> tlogFilesToDownload; + + private volatile List> contentStoreFilesToDownload; + + private volatile List> filesDownloaded; + + private volatile List> confFilesDownloaded; + + private volatile List> tlogFilesDownloaded; + + private volatile List> contentStoreFilesDownloaded; + + private volatile Map currentFile; + + private volatile DirectoryFileFetcher dirFileFetcher; + + private volatile LocalFsFileFetcher localFileFetcher; + + private volatile ContentStoreFetcher contentStoreFileFetcher; + + private volatile ExecutorService fsyncService; + + private volatile boolean stop = false; + + private boolean useInternalCompression = false; + + private boolean useExternalCompression = false; + + private final HttpClient myHttpClient; + + private static final String INTERRUPT_RESPONSE_MESSAGE = "Interrupted while waiting for modify lock"; + + + public static class IndexFetchResult { + private final String message; + private final boolean successful; + private final Throwable exception; + + public static final String FAILED_BY_INTERRUPT_MESSAGE = "Fetching index failed by interrupt"; + public static final String FAILED_BY_EXCEPTION_MESSAGE = "Fetching index failed by exception"; + + /** pre-defined results */ + public static final IndexFetchResult ALREADY_IN_SYNC = new IndexFetchResult("Local index commit is already in sync with peer", true, null); + public static final IndexFetchResult INDEX_FETCH_FAILURE = new IndexFetchResult("Fetching lastest index is failed", false, null); + public static final IndexFetchResult INDEX_FETCH_SUCCESS = new IndexFetchResult("Fetching latest index is successful", true, null); + public static final IndexFetchResult LOCK_OBTAIN_FAILED = new IndexFetchResult("Obtaining SnapPuller lock failed", false, null); + public static final IndexFetchResult MASTER_VERSION_ZERO = new IndexFetchResult("Index in peer is empty and never committed yet", true, null); + public static final IndexFetchResult NO_INDEX_COMMIT_EXIST = new IndexFetchResult("No IndexCommit in local index", false, null); + public static final IndexFetchResult PEER_INDEX_COMMIT_DELETED = new IndexFetchResult("No files to download because IndexCommit in peer was deleted", false, null); + public static final IndexFetchResult LOCAL_ACTIVITY_DURING_REPLICATION = new IndexFetchResult("Local index modification during replication", false, null); + + IndexFetchResult(String message, boolean successful, Throwable exception) { + this.message = message; + this.successful = successful; + this.exception = exception; + } + + /* + * @return exception thrown if failed by exception or interrupt, otherwise null + */ + public Throwable getException() { + return this.exception; + } + + /* + * @return true if index fetch was successful, false otherwise + */ + public boolean getSuccessful() { + return this.successful; + } + + public String getMessage() { + return this.message; + } + } + + private static HttpClient createHttpClient(SolrCore core, String connTimeout, String readTimeout, String httpBasicAuthUser, String httpBasicAuthPassword, boolean useCompression) { + final ModifiableSolrParams httpClientParams = new ModifiableSolrParams(); + httpClientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connTimeout != null ? connTimeout : "5000"); + httpClientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, readTimeout != null ? readTimeout : "20000"); + httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_USER, httpBasicAuthUser); + httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword); + httpClientParams.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, useCompression); + + return HttpClientUtil.createClient(httpClientParams, core.getCoreContainer().getUpdateShardHandler().getConnectionManager()); + } + + public IndexFetcher(final NamedList initArgs, final ReplicationHandler handler, final SolrCore sc) { + solrCore = sc; + String masterUrl = (String) initArgs.get(MASTER_URL); + if (masterUrl == null) + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "'masterUrl' is required for a slave"); + if (masterUrl.endsWith(ReplicationHandler.PATH)) { + masterUrl = masterUrl.substring(0, masterUrl.length()-12); + LOG.warn("'masterUrl' must be specified without the "+ ReplicationHandler.PATH+" suffix"); + } + this.masterUrl = masterUrl; + + this.replicationHandler = handler; + String compress = (String) initArgs.get(COMPRESSION); + useInternalCompression = INTERNAL.equals(compress); + useExternalCompression = EXTERNAL.equals(compress); + String connTimeout = (String) initArgs.get(HttpClientUtil.PROP_CONNECTION_TIMEOUT); + // allow a master override for tests - you specify this in /replication slave section of solrconfig and some + // test don't want to define this + String readTimeout = System.getProperty("solr.indexfetcher.sotimeout", "-1"); + if (readTimeout.equals("-1")) { + readTimeout = (String) initArgs.get(HttpClientUtil.PROP_SO_TIMEOUT); + } + String httpBasicAuthUser = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_USER); + String httpBasicAuthPassword = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_PASS); + myHttpClient = createHttpClient(solrCore, connTimeout, readTimeout, httpBasicAuthUser, httpBasicAuthPassword, useExternalCompression); + } + + /** + * Gets the latest commit version and generation from the master + */ + @SuppressWarnings("unchecked") + NamedList getLatestVersion() throws IOException { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(COMMAND, CMD_INDEX_VERSION); + params.set(CommonParams.WT, JAVABIN); + params.set(CommonParams.QT, ReplicationHandler.PATH); + QueryRequest req = new QueryRequest(params); + + // TODO modify to use shardhandler + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) { + client.setSoTimeout(60000); + client.setConnectionTimeout(15000); + + return client.request(req); + } catch (SolrServerException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, e.getMessage(), e); + } + } + + /** + * Fetches the list of files in a given index commit point and updates internal list of files to download. + */ + private void fetchFileList(long gen) throws IOException { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(COMMAND, CMD_GET_FILE_LIST); + params.set(GENERATION, String.valueOf(gen)); + params.set(CommonParams.WT, JAVABIN); + params.set(CommonParams.QT, ReplicationHandler.PATH); + QueryRequest req = new QueryRequest(params); + + // TODO modify to use shardhandler + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) { + client.setSoTimeout(60000); + client.setConnectionTimeout(15000); + NamedList response = client.request(req); + + List> files = (List>) response.get(CMD_GET_FILE_LIST); + if (files != null) + filesToDownload = Collections.synchronizedList(files); + else { + filesToDownload = Collections.emptyList(); + LOG.error("No files to download for index generation: "+ gen); + } + + files = (List>) response.get(CONF_FILES); + if (files != null) + confFilesToDownload = Collections.synchronizedList(files); + + files = (List>) response.get(TLOG_FILES); + if (files != null) { + tlogFilesToDownload = Collections.synchronizedList(files); + } + + files = (List>) response.get(CONTENT_STORE_FILES); + + if (files != null) { + contentStoreFilesToDownload = Collections.synchronizedList(files); + } + + } catch (SolrServerException e) { + throw new IOException(e); + } + } + + IndexFetchResult fetchLatestIndex(boolean forceReplication) throws IOException, InterruptedException { + return fetchLatestIndex(forceReplication, false); + } + + /** + * This command downloads all the necessary files from master to install a index commit point. Only changed files are + * downloaded. It also downloads the conf files (if they are modified). + * + * @param forceReplication force a replication in all cases + * @param forceCoreReload force a core reload in all cases + * @return true on success, false if slave is already in sync + * @throws IOException if an exception occurs + */ + IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreReload) throws IOException, InterruptedException { + + boolean cleanupDone = false; + boolean successfulInstall = false; + markReplicationStart(); + Directory tmpIndexDir = null; + String tmpIndex; + Directory indexDir = null; + String indexDirPath; + boolean deleteTmpIdxDir = true; + File tmpTlogDir = null; + + File tmpContentStoreDir = null; + + if (!solrCore.getSolrCoreState().getLastReplicateIndexSuccess()) { + // if the last replication was not a success, we force a full replication + // when we are a bit more confident we may want to try a partial replication + // if the error is connection related or something, but we have to be careful + forceReplication = true; + } + + try { + //get the current 'replicateable' index version in the master + NamedList response; + try { + response = getLatestVersion(); + } catch (Exception e) { + final String errorMsg = e.toString(); + if (!Strings.isNullOrEmpty(errorMsg) && errorMsg.contains(INTERRUPT_RESPONSE_MESSAGE)) { + LOG.warn("Master at: " + masterUrl + " is not available. Index fetch failed by interrupt. Exception: " + errorMsg); + return new IndexFetchResult(IndexFetchResult.FAILED_BY_INTERRUPT_MESSAGE, false, e); + } else { + LOG.warn("Master at: " + masterUrl + " is not available. Index fetch failed by exception: " + errorMsg); + return new IndexFetchResult(IndexFetchResult.FAILED_BY_EXCEPTION_MESSAGE, false, e); + } + } + + long latestVersion = (Long) response.get(CMD_INDEX_VERSION); + long latestGeneration = (Long) response.get(GENERATION); + + LOG.info("Master's generation: " + latestGeneration); + LOG.info("Master's version: " + latestVersion); + + // TODO: make sure that getLatestCommit only returns commit points for the main index (i.e. no side-car indexes) + IndexCommit commit = solrCore.getDeletionPolicy().getLatestCommit(); + if (commit == null) { + // Presumably the IndexWriter hasn't been opened yet, and hence the deletion policy hasn't been updated with commit points + RefCounted searcherRefCounted = null; + try { + searcherRefCounted = solrCore.getNewestSearcher(false); + if (searcherRefCounted == null) { + LOG.warn("No open searcher found - fetch aborted"); + return IndexFetchResult.NO_INDEX_COMMIT_EXIST; + } + commit = searcherRefCounted.get().getIndexReader().getIndexCommit(); + } finally { + if (searcherRefCounted != null) + searcherRefCounted.decref(); + } + } + + LOG.info("Slave's generation: " + commit.getGeneration()); + LOG.info("Slave's version: " + IndexDeletionPolicyWrapper.getCommitTimestamp(commit)); + + if (latestVersion == 0L) { + if (forceReplication && commit.getGeneration() != 0) { + // since we won't get the files for an empty index, + // we just clear ours and commit + RefCounted iw = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore); + try { + iw.get().deleteAll(); + } finally { + iw.decref(); + } + SolrQueryRequest req = new LocalSolrQueryRequest(solrCore, new ModifiableSolrParams()); + solrCore.getUpdateHandler().commit(new CommitUpdateCommand(req, false)); + } + + //there is nothing to be replicated + successfulInstall = true; + return IndexFetchResult.MASTER_VERSION_ZERO; + } + + // TODO: Should we be comparing timestamps (across machines) here? + if (!forceReplication && IndexDeletionPolicyWrapper.getCommitTimestamp(commit) == latestVersion) { + //master and slave are already in sync just return + LOG.info("Slave in sync with master."); + successfulInstall = true; + return IndexFetchResult.ALREADY_IN_SYNC; + } + LOG.info("Starting replication process"); + // get the list of files first + fetchFileList(latestGeneration); + // this can happen if the commit point is deleted before we fetch the file list. + if (filesToDownload.isEmpty()) { + return IndexFetchResult.PEER_INDEX_COMMIT_DELETED; + } + LOG.info("Number of files in latest index in master: " + filesToDownload.size()); + if (tlogFilesToDownload != null) { + LOG.info("Number of tlog files in master: " + tlogFilesToDownload.size()); + } + + // Create the sync service + fsyncService = ExecutorUtil.newMDCAwareSingleThreadExecutor(new DefaultSolrThreadFactory("fsyncService")); + // use a synchronized list because the list is read by other threads (to show details) + filesDownloaded = Collections.synchronizedList(new ArrayList>()); + // if the generation of master is older than that of the slave , it means they are not compatible to be copied + // then a new index directory to be created and all the files need to be copied + boolean isFullCopyNeeded = IndexDeletionPolicyWrapper + .getCommitTimestamp(commit) >= latestVersion + || commit.getGeneration() >= latestGeneration || forceReplication; + + String timestamp = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date()); + String tmpIdxDirName = "index." + timestamp; + tmpIndex = solrCore.getDataDir() + tmpIdxDirName; + + tmpIndexDir = solrCore.getDirectoryFactory().get(tmpIndex, DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); + + // tmp dir for tlog files + if (tlogFilesToDownload != null) { + tmpTlogDir = new File(solrCore.getUpdateHandler().getUpdateLog().getLogDir(), "tlog." + timestamp); + } + + // cindex dir... + indexDirPath = solrCore.getIndexDir(); + indexDir = solrCore.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); + + try { + + //We will compare all the index files from the master vs the index files on disk to see if there is a mismatch + //in the metadata. If there is a mismatch for the same index file then we download the entire index again. + if (!isFullCopyNeeded && isIndexStale(indexDir)) { + isFullCopyNeeded = true; + } + + if (!isFullCopyNeeded) { + // a searcher might be using some flushed but not committed segments + // because of soft commits (which open a searcher on IW's data) + // so we need to close the existing searcher on the last commit + // and wait until we are able to clean up all unused lucene files + if (solrCore.getCoreContainer().isZooKeeperAware()) { + solrCore.closeSearcher(); + } + + // rollback and reopen index writer and wait until all unused files + // are successfully deleted + solrCore.getUpdateHandler().newIndexWriter(true); + RefCounted writer = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null); + try { + IndexWriter indexWriter = writer.get(); + int c = 0; + indexWriter.deleteUnusedFiles(); + while (hasUnusedFiles(indexDir, commit)) { + indexWriter.deleteUnusedFiles(); + LOG.info("Sleeping for 1000ms to wait for unused lucene index files to be delete-able"); + Thread.sleep(1000); + c++; + if (c >= 30) { + LOG.warn("IndexFetcher unable to cleanup unused lucene index files so we must do a full copy instead"); + isFullCopyNeeded = true; + break; + } + } + if (c > 0) { + LOG.info("IndexFetcher slept for " + (c * 1000) + "ms for unused lucene index files to be delete-able"); + } + } finally { + writer.decref(); + } + } + boolean reloadCore = false; + + try { + // we have to be careful and do this after we know isFullCopyNeeded won't be flipped + if (!isFullCopyNeeded) { + solrCore.getUpdateHandler().getSolrCoreState().closeIndexWriter(solrCore, true); + } + + LOG.info("Starting download (fullCopy={}) to {}", isFullCopyNeeded, tmpIndexDir); + successfulInstall = false; + + + long local = downloadContentStoreFiles(new File(ContentStoreCache.get().getContentStoreRootPath()), 0L); + + + long bytesDownloaded = downloadIndexFiles(isFullCopyNeeded, indexDir, tmpIndexDir, latestGeneration); + if (tlogFilesToDownload != null) { + bytesDownloaded += downloadTlogFiles(tmpTlogDir, latestGeneration); + reloadCore = true; // reload update log + } + final long timeTakenSeconds = getReplicationTimeElapsed(); + final Long bytesDownloadedPerSecond = (timeTakenSeconds != 0 ? new Long(bytesDownloaded/timeTakenSeconds) : null); + LOG.info("Total time taken for download (fullCopy={},bytesDownloaded={}) : {} secs ({} bytes/sec) to {}", + new Object[]{isFullCopyNeeded, bytesDownloaded, timeTakenSeconds, bytesDownloadedPerSecond, tmpIndexDir}); + + Collection> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload); + if (!modifiedConfFiles.isEmpty()) { + reloadCore = true; + downloadConfFiles(confFilesToDownload, latestGeneration); + if (isFullCopyNeeded) { + successfulInstall = solrCore.modifyIndexProps(tmpIdxDirName); + if (successfulInstall) deleteTmpIdxDir = false; + } else { + successfulInstall = moveIndexFiles(tmpIndexDir, indexDir); + } + if (tlogFilesToDownload != null) { + // move tlog files and refresh ulog only if we successfully installed a new index + successfulInstall &= moveTlogFiles(tmpTlogDir); + } + if (successfulInstall) { + if (isFullCopyNeeded) { + // let the system know we are changing dir's and the old one + // may be closed + if (indexDir != null) { + solrCore.getDirectoryFactory().doneWithDirectory(indexDir); + // Cleanup all index files not associated with any *named* snapshot. + solrCore.deleteNonSnapshotIndexFiles(indexDirPath); + } + } + + LOG.info("Configuration files are modified, core will be reloaded"); + logReplicationTimeAndConfFiles(modifiedConfFiles, + successfulInstall);// write to a file time of replication and + // conf files. + } + } else { + terminateAndWaitFsyncService(); + if (isFullCopyNeeded) { + successfulInstall = solrCore.modifyIndexProps(tmpIdxDirName); + if (successfulInstall) deleteTmpIdxDir = false; + } else { + successfulInstall = moveIndexFiles(tmpIndexDir, indexDir); + } + if (tlogFilesToDownload != null) { + // move tlog files and refresh ulog only if we successfully installed a new index + successfulInstall &= moveTlogFiles(tmpTlogDir); + } + if (successfulInstall) { + logReplicationTimeAndConfFiles(modifiedConfFiles, + successfulInstall); + } + } + } finally { + if (!isFullCopyNeeded) { + solrCore.getUpdateHandler().getSolrCoreState().openIndexWriter(solrCore); + } + } + + // we must reload the core after we open the IW back up + if (successfulInstall && (reloadCore || forceCoreReload)) { + LOG.info("Reloading SolrCore {}", solrCore.getName()); + reloadCore(); + } + + if (successfulInstall) { + if (isFullCopyNeeded) { + // let the system know we are changing dir's and the old one + // may be closed + if (indexDir != null) { + LOG.info("removing old index directory " + indexDir); + solrCore.getDirectoryFactory().doneWithDirectory(indexDir); + solrCore.getDirectoryFactory().remove(indexDir); + } + } + if (isFullCopyNeeded) { + solrCore.getUpdateHandler().newIndexWriter(isFullCopyNeeded); + } + + openNewSearcherAndUpdateCommitPoint(); + } + + if (!isFullCopyNeeded && !forceReplication && !successfulInstall) { + cleanup(solrCore, tmpIndexDir, indexDir, deleteTmpIdxDir, tmpTlogDir, successfulInstall); + cleanupDone = true; + // we try with a full copy of the index + LOG.warn( + "Replication attempt was not successful - trying a full index replication reloadCore={}", + reloadCore); + successfulInstall = fetchLatestIndex(true, reloadCore).getSuccessful(); + } + + markReplicationStop(); + return successfulInstall ? IndexFetchResult.INDEX_FETCH_SUCCESS : IndexFetchResult.INDEX_FETCH_FAILURE; + } catch (ReplicationHandlerException e) { + LOG.error("User aborted Replication"); + return new IndexFetchResult(IndexFetchResult.FAILED_BY_EXCEPTION_MESSAGE, false, e); + } catch (SolrException e) { + throw e; + } catch (InterruptedException e) { + throw new InterruptedException("Index fetch interrupted"); + } catch (Exception e) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Index fetch failed : ", e); + } + } finally { + if (!cleanupDone) { + cleanup(solrCore, tmpIndexDir, indexDir, deleteTmpIdxDir, tmpTlogDir, successfulInstall); + } + } + } + + + + /** + * Download all the tlog files to the temp tlog directory. + */ + private long downloadContentStoreFiles(File contentStoreDirectory, long latestGeneration) throws Exception { + LOG.info("Starting download of tlog files from master: " + tlogFilesToDownload); + contentStoreFilesDownloaded = Collections.synchronizedList(new ArrayList<>()); + long bytesDownloaded = 0; + + for (List> partition : Lists.partition(contentStoreFilesToDownload, 50)) { + contentStoreFileFetcher = new ContentStoreFetcher( + contentStoreDirectory, + ReplicationHandler.CONTENT_STORE_FILES, + latestGeneration, + partition); + + contentStoreFileFetcher.fetchFile(); // TODO change name + bytesDownloaded += contentStoreFileFetcher.getBytesDownloaded(); + contentStoreFilesDownloaded.addAll(partition); + } + return bytesDownloaded; + } + + + private void cleanup(final SolrCore core, Directory tmpIndexDir, + Directory indexDir, boolean deleteTmpIdxDir, File tmpTlogDir, boolean successfulInstall) throws IOException { + try { + if (!successfulInstall) { + try { + logReplicationTimeAndConfFiles(null, successfulInstall); + } catch (Exception e) { + // this can happen on shutdown, a fetch may be running in a thread after DirectoryFactory is closed + LOG.warn("Could not log failed replication details", e); + } + } + + if (core.getCoreContainer().isZooKeeperAware()) { + // we only track replication success in SolrCloud mode + core.getUpdateHandler().getSolrCoreState().setLastReplicateIndexSuccess(successfulInstall); + } + + filesToDownload = filesDownloaded = confFilesDownloaded = confFilesToDownload = tlogFilesToDownload + = tlogFilesDownloaded = contentStoreFilesDownloaded = contentStoreFilesToDownload = null; + markReplicationStop(); + dirFileFetcher = null; + localFileFetcher = null; + if (fsyncService != null && !fsyncService.isShutdown()) fsyncService.shutdown(); + fsyncService = null; + stop = false; + fsyncException = null; + } finally { + // order below is important + try { + if (tmpIndexDir != null && deleteTmpIdxDir) { + core.getDirectoryFactory().doneWithDirectory(tmpIndexDir); + core.getDirectoryFactory().remove(tmpIndexDir); + } + } catch (Exception e) { + SolrException.log(LOG, e); + } finally { + try { + if (tmpIndexDir != null) core.getDirectoryFactory().release(tmpIndexDir); + } catch (Exception e) { + SolrException.log(LOG, e); + } + try { + if (indexDir != null) { + core.getDirectoryFactory().release(indexDir); + } + } catch (Exception e) { + SolrException.log(LOG, e); + } + try { + if (tmpTlogDir != null) delTree(tmpTlogDir); + } catch (Exception e) { + SolrException.log(LOG, e); + } + } + } + } + + private boolean hasUnusedFiles(Directory indexDir, IndexCommit commit) throws IOException { + String segmentsFileName = commit.getSegmentsFileName(); + SegmentInfos infos = SegmentInfos.readCommit(indexDir, segmentsFileName); + Set currentFiles = new HashSet<>(infos.files(true)); + String[] allFiles = indexDir.listAll(); + for (String file : allFiles) { + if (!file.equals(segmentsFileName) && !currentFiles.contains(file) && !file.endsWith(".lock")) { + LOG.info("Found unused file: " + file); + return true; + } + } + return false; + } + + private volatile Exception fsyncException; + + /** + * terminate the fsync service and wait for all the tasks to complete. If it is already terminated + */ + private void terminateAndWaitFsyncService() throws Exception { + if (fsyncService.isTerminated()) return; + fsyncService.shutdown(); + // give a long wait say 1 hr + fsyncService.awaitTermination(3600, TimeUnit.SECONDS); + // if any fsync failed, throw that exception back + Exception fsyncExceptionCopy = fsyncException; + if (fsyncExceptionCopy != null) throw fsyncExceptionCopy; + } + + /** + * Helper method to record the last replication's details so that we can show them on the statistics page across + * restarts. + * @throws IOException on IO error + */ + @SuppressForbidden(reason = "Need currentTimeMillis for debugging/stats") + private void logReplicationTimeAndConfFiles(Collection> modifiedConfFiles, boolean successfulInstall) throws IOException { + List confFiles = new ArrayList<>(); + if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty()) + for (Map map1 : modifiedConfFiles) + confFiles.add((String) map1.get(NAME)); + + Properties props = replicationHandler.loadReplicationProperties(); + long replicationTime = System.currentTimeMillis(); + long replicationTimeTaken = getReplicationTimeElapsed(); + Directory dir = null; + try { + dir = solrCore.getDirectoryFactory().get(solrCore.getDataDir(), DirContext.META_DATA, solrCore.getSolrConfig().indexConfig.lockType); + + int indexCount = 1, confFilesCount = 1; + if (props.containsKey(TIMES_INDEX_REPLICATED)) { + indexCount = Integer.parseInt(props.getProperty(TIMES_INDEX_REPLICATED)) + 1; + } + StringBuilder sb = readToStringBuilder(replicationTime, props.getProperty(INDEX_REPLICATED_AT_LIST)); + props.setProperty(INDEX_REPLICATED_AT_LIST, sb.toString()); + props.setProperty(INDEX_REPLICATED_AT, String.valueOf(replicationTime)); + props.setProperty(PREVIOUS_CYCLE_TIME_TAKEN, String.valueOf(replicationTimeTaken)); + props.setProperty(TIMES_INDEX_REPLICATED, String.valueOf(indexCount)); + if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty()) { + props.setProperty(CONF_FILES_REPLICATED, confFiles.toString()); + props.setProperty(CONF_FILES_REPLICATED_AT, String.valueOf(replicationTime)); + if (props.containsKey(TIMES_CONFIG_REPLICATED)) { + confFilesCount = Integer.parseInt(props.getProperty(TIMES_CONFIG_REPLICATED)) + 1; + } + props.setProperty(TIMES_CONFIG_REPLICATED, String.valueOf(confFilesCount)); + } + + props.setProperty(LAST_CYCLE_BYTES_DOWNLOADED, String.valueOf(getTotalBytesDownloaded())); + if (!successfulInstall) { + int numFailures = 1; + if (props.containsKey(TIMES_FAILED)) { + numFailures = Integer.parseInt(props.getProperty(TIMES_FAILED)) + 1; + } + props.setProperty(TIMES_FAILED, String.valueOf(numFailures)); + props.setProperty(REPLICATION_FAILED_AT, String.valueOf(replicationTime)); + sb = readToStringBuilder(replicationTime, props.getProperty(REPLICATION_FAILED_AT_LIST)); + props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString()); + } + + + String tmpFileName = REPLICATION_PROPERTIES + "." + System.nanoTime(); + final IndexOutput out = dir.createOutput(tmpFileName, DirectoryFactory.IOCONTEXT_NO_CACHE); + Writer outFile = new OutputStreamWriter(new PropertiesOutputStream(out), StandardCharsets.UTF_8); + try { + props.store(outFile, "Replication details"); + dir.sync(Collections.singleton(tmpFileName)); + } finally { + IOUtils.closeQuietly(outFile); + } + + solrCore.getDirectoryFactory().renameWithOverwrite(dir, tmpFileName, REPLICATION_PROPERTIES); + } catch (Exception e) { + LOG.warn("Exception while updating statistics", e); + } finally { + if (dir != null) { + solrCore.getDirectoryFactory().release(dir); + } + } + } + + long getTotalBytesDownloaded() { + long bytesDownloaded = 0; + //get size from list of files to download + for (Map file : getFilesDownloaded()) { + bytesDownloaded += (Long) file.get(SIZE); + } + + //get size from list of conf files to download + for (Map file : getConfFilesDownloaded()) { + bytesDownloaded += (Long) file.get(SIZE); + } + + //get size from current file being downloaded + Map currentFile = getCurrentFile(); + if (currentFile != null) { + if (currentFile.containsKey("bytesDownloaded")) { + bytesDownloaded += (Long) currentFile.get("bytesDownloaded"); + } + } + return bytesDownloaded; + } + + private StringBuilder readToStringBuilder(long replicationTime, String str) { + StringBuilder sb = new StringBuilder(); + List l = new ArrayList<>(); + if (str != null && str.length() != 0) { + String[] ss = str.split(","); + Collections.addAll(l, ss); + } + sb.append(replicationTime); + if (!l.isEmpty()) { + for (int i = 0; i < l.size() || i < 9; i++) { + if (i == l.size() || i == 9) break; + String s = l.get(i); + sb.append(",").append(s); + } + } + return sb; + } + + private void openNewSearcherAndUpdateCommitPoint() throws IOException { + RefCounted searcher = null; + IndexCommit commitPoint; + // must get the latest solrCore object because the one we have might be closed because of a reload + // todo stop keeping solrCore around + SolrCore core = solrCore.getCoreContainer().getCore(solrCore.getName()); + try { + Future[] waitSearcher = new Future[1]; + searcher = core.getSearcher(true, true, waitSearcher, true); + if (waitSearcher[0] != null) { + try { + waitSearcher[0].get(); + } catch (InterruptedException | ExecutionException e) { + SolrException.log(LOG, e); + } + } + commitPoint = searcher.get().getIndexReader().getIndexCommit(); + } finally { + if (searcher != null) { + searcher.decref(); + } + core.close(); + } + + // update the commit point in replication handler + replicationHandler.indexCommitPoint = commitPoint; + + } + + private void reloadCore() { + final CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + try { + solrCore.getCoreContainer().reload(solrCore.getName()); + } catch (Exception e) { + LOG.error("Could not reload core ", e); + } finally { + latch.countDown(); + } + }).start(); + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting for core reload to finish", e); + } + } + + private void downloadConfFiles(List> confFilesToDownload, long latestGeneration) throws Exception { + LOG.info("Starting download of configuration files from master: " + confFilesToDownload); + confFilesDownloaded = Collections.synchronizedList(new ArrayList<>()); + File tmpconfDir = new File(solrCore.getResourceLoader().getConfigDir(), "conf." + getDateAsStr(new Date())); + try { + boolean status = tmpconfDir.mkdirs(); + if (!status) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to create temporary config folder: " + tmpconfDir.getName()); + } + for (Map file : confFilesToDownload) { + String saveAs = (String) (file.get(ALIAS) == null ? file.get(NAME) : file.get(ALIAS)); + localFileFetcher = new LocalFsFileFetcher(tmpconfDir, file, saveAs, CONF_FILE_SHORT, latestGeneration); + currentFile = file; + localFileFetcher.fetchFile(); + confFilesDownloaded.add(new HashMap<>(file)); + } + // this is called before copying the files to the original conf dir + // so that if there is an exception avoid corrupting the original files. + terminateAndWaitFsyncService(); + copyTmpConfFiles2Conf(tmpconfDir); + } finally { + delTree(tmpconfDir); + } + } + + /** + * Download all the tlog files to the temp tlog directory. + */ + private long downloadTlogFiles(File tmpTlogDir, long latestGeneration) throws Exception { + LOG.info("Starting download of tlog files from master: " + tlogFilesToDownload); + tlogFilesDownloaded = Collections.synchronizedList(new ArrayList<>()); + long bytesDownloaded = 0; + + boolean status = tmpTlogDir.mkdirs(); + if (!status) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to create temporary tlog folder: " + tmpTlogDir.getName()); + } + for (Map file : tlogFilesToDownload) { + String saveAs = (String) (file.get(ALIAS) == null ? file.get(NAME) : file.get(ALIAS)); + localFileFetcher = new LocalFsFileFetcher(tmpTlogDir, file, saveAs, TLOG_FILE, latestGeneration); + currentFile = file; + localFileFetcher.fetchFile(); + bytesDownloaded += localFileFetcher.getBytesDownloaded(); + tlogFilesDownloaded.add(new HashMap<>(file)); + } + return bytesDownloaded; + } + + + + /** + * Download the index files. If a new index is needed, download all the files. + * + * @param downloadCompleteIndex is it a fresh index copy + * @param tmpIndexDir the directory to which files need to be downloadeed to + * @param indexDir the indexDir to be merged to + * @param latestGeneration the version number + * + * @return number of bytes downloaded + */ + private long downloadIndexFiles(boolean downloadCompleteIndex, Directory indexDir, Directory tmpIndexDir, long latestGeneration) + throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("Download files to dir: " + Arrays.asList(indexDir.listAll())); + } + long bytesDownloaded = 0; + for (Map file : filesToDownload) { + String filename = (String) file.get(NAME); + long size = (Long) file.get(SIZE); + CompareResult compareResult = compareFile(indexDir, filename, size, (Long) file.get(CHECKSUM)); + boolean alwaysDownload = filesToAlwaysDownloadIfNoChecksums(filename, size, compareResult); + LOG.debug("Downloading file={} size={} checksum={} alwaysDownload={}", new Object[]{filename, size, file.get(CHECKSUM), alwaysDownload}); + if (!compareResult.equal || downloadCompleteIndex || alwaysDownload) { + dirFileFetcher = new DirectoryFileFetcher(tmpIndexDir, file, + (String) file.get(NAME), FILE, latestGeneration); + currentFile = file; + dirFileFetcher.fetchFile(); + bytesDownloaded += dirFileFetcher.getBytesDownloaded(); + filesDownloaded.add(new HashMap<>(file)); + } else { + LOG.info("Skipping download for " + file.get(NAME) + + " because it already exists"); + } + } + return bytesDownloaded; + } + + static boolean filesToAlwaysDownloadIfNoChecksums(String filename, + long size, CompareResult compareResult) { + // without checksums to compare, we always download .si, .liv, segments_N, + // and any very small files + return !compareResult.checkSummed && (filename.endsWith(".si") || filename.endsWith(".liv") + || filename.startsWith("segments_") || size < _100K); + } + + protected static class CompareResult { + boolean equal = false; + boolean checkSummed = false; + } + + protected static CompareResult compareFile(Directory indexDir, String filename, Long backupIndexFileLen, Long backupIndexFileChecksum) { + CompareResult compareResult = new CompareResult(); + try { + try (final IndexInput indexInput = indexDir.openInput(filename, IOContext.READONCE)) { + long indexFileLen = indexInput.length(); + long indexFileChecksum = 0; + + if (backupIndexFileChecksum != null) { + try { + indexFileChecksum = CodecUtil.retrieveChecksum(indexInput); + compareResult.checkSummed = true; + } catch (Exception e) { + LOG.warn("Could not retrieve checksum from file.", e); + } + } + + if (!compareResult.checkSummed) { + // we don't have checksums to compare + + if (indexFileLen == backupIndexFileLen) { + compareResult.equal = true; + return compareResult; + } else { + LOG.info( + "File {} did not match. expected length is {} and actual length is {}", new Object[]{filename, backupIndexFileLen, indexFileLen}); + compareResult.equal = false; + return compareResult; + } + } + + // we have checksums to compare + + if (indexFileLen == backupIndexFileLen && indexFileChecksum == backupIndexFileChecksum) { + compareResult.equal = true; + return compareResult; + } else { + LOG.warn("File {} did not match. expected checksum is {} and actual is checksum {}. " + + "expected length is {} and actual length is {}", new Object[]{filename, backupIndexFileChecksum, indexFileChecksum, + backupIndexFileLen, indexFileLen}); + compareResult.equal = false; + return compareResult; + } + } + } catch (NoSuchFileException | FileNotFoundException e) { + compareResult.equal = false; + return compareResult; + } catch (IOException e) { + LOG.error("Could not read file " + filename + ". Downloading it again", e); + compareResult.equal = false; + return compareResult; + } + } + + /** Returns true if the file exists (can be opened), false + * if it cannot be opened, and (unlike Java's + * File.exists) throws IOException if there's some + * unexpected error. */ + private static boolean slowFileExists(Directory dir, String fileName) throws IOException { + try { + dir.openInput(fileName, IOContext.DEFAULT).close(); + return true; + } catch (NoSuchFileException | FileNotFoundException e) { + return false; + } + } + + /** + * All the files which are common between master and slave must have same size and same checksum else we assume + * they are not compatible (stale). + * + * @return true if the index stale and we need to download a fresh copy, false otherwise. + * @throws IOException if low level io error + */ + private boolean isIndexStale(Directory dir) throws IOException { + for (Map file : filesToDownload) { + String filename = (String) file.get(NAME); + Long length = (Long) file.get(SIZE); + Long checksum = (Long) file.get(CHECKSUM); + if (slowFileExists(dir, filename)) { + if (checksum != null) { + if (!(compareFile(dir, filename, length, checksum).equal)) { + // file exists and size or checksum is different, therefore we must download it again + return true; + } + } else { + if (length != dir.fileLength(filename)) { + LOG.warn("File {} did not match. expected length is {} and actual length is {}",new Object[]{ + filename, length, dir.fileLength(filename)}); + return true; + } + } + } + } + return false; + } + + /** + * Copy a file by the File#renameTo() method. If it fails, it is considered a failure + *

+ */ + private boolean moveAFile(Directory tmpIdxDir, Directory indexDir, String fname) { + LOG.debug("Moving file: {}", fname); + boolean success = false; + try { + if (slowFileExists(indexDir, fname)) { + LOG.warn("Cannot complete replication attempt because file already exists:" + fname); + + // we fail - we downloaded the files we need, if we can't move one in, we can't + // count on the correct index + return false; + } + } catch (IOException e) { + SolrException.log(LOG, "could not check if a file exists", e); + return false; + } + try { + solrCore.getDirectoryFactory().move(tmpIdxDir, indexDir, fname, DirectoryFactory.IOCONTEXT_NO_CACHE); + success = true; + } catch (IOException e) { + SolrException.log(LOG, "Could not move file", e); + } + return success; + } + + /** + * Copy all index files from the temp index dir to the actual index. The segments_N file is copied last. + */ + private boolean moveIndexFiles(Directory tmpIdxDir, Directory indexDir) { + if (LOG.isDebugEnabled()) { + try { + LOG.info("From dir files:" + Arrays.asList(tmpIdxDir.listAll())); + LOG.info("To dir files:" + Arrays.asList(indexDir.listAll())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + String segmentsFile = null; + for (Map f : filesDownloaded) { + String fname = (String) f.get(NAME); + // the segments file must be copied last + // or else if there is a failure in between the + // index will be corrupted + if (fname.startsWith("segments_")) { + //The segments file must be copied in the end + //Otherwise , if the copy fails index ends up corrupted + segmentsFile = fname; + continue; + } + if (!moveAFile(tmpIdxDir, indexDir, fname)) return false; + } + //copy the segments file last + if (segmentsFile != null) { + if (!moveAFile(tmpIdxDir, indexDir, segmentsFile)) return false; + } + return true; + } + + /** + *

+ * Copy all the tlog files from the temp tlog dir to the actual tlog dir, and reset + * the {@link UpdateLog}. The copy will try to preserve the original tlog directory + * if the copy fails. + *

+ *

+ * This assumes that the tlog files transferred from the leader are in synch with the + * index files transferred from the leader. The reset of the update log relies on the version + * of the latest operations found in the tlog files. If the tlogs are ahead of the latest commit + * point, it will not copy all the needed buffered updates for the replay and it will miss + * some operations. + *

+ */ + private boolean moveTlogFiles(File tmpTlogDir) { + UpdateLog ulog = solrCore.getUpdateHandler().getUpdateLog(); + + VersionInfo vinfo = ulog.getVersionInfo(); + vinfo.blockUpdates(); // block updates until the new update log is initialised + try { + // reset the update log before copying the new tlog directory + CdcrUpdateLog.BufferedUpdates bufferedUpdates = ((CdcrUpdateLog) ulog).resetForRecovery(); + // try to move the temp tlog files to the tlog directory + if (!copyTmpTlogFiles2Tlog(tmpTlogDir)) return false; + // reinitialise the update log and copy the buffered updates + if (bufferedUpdates.tlog != null) { + // map file path to its new backup location + File parentDir = FileSystems.getDefault().getPath(solrCore.getUpdateHandler().getUpdateLog().getLogDir()).getParent().toFile(); + File backupTlogDir = new File(parentDir, tmpTlogDir.getName()); + bufferedUpdates.tlog = new File(backupTlogDir, bufferedUpdates.tlog.getName()); + } + // init the update log with the new set of tlog files, and copy the buffered updates + ((CdcrUpdateLog) ulog).initForRecovery(bufferedUpdates.tlog, bufferedUpdates.offset); + } + catch (Exception e) { + LOG.error("Unable to copy tlog files", e); + return false; + } + finally { + vinfo.unblockUpdates(); + } + return true; + } + + /** + * Make file list + */ + private List makeTmpConfDirFileList(File dir, List fileList) { + File[] files = dir.listFiles(); + for (File file : files) { + if (file.isFile()) { + fileList.add(file); + } else if (file.isDirectory()) { + fileList = makeTmpConfDirFileList(file, fileList); + } + } + return fileList; + } + + /** + * The conf files are copied to the tmp dir to the conf dir. A backup of the old file is maintained + */ + private void copyTmpConfFiles2Conf(File tmpconfDir) { + boolean status = false; + File confDir = new File(solrCore.getResourceLoader().getConfigDir()); + for (File file : makeTmpConfDirFileList(tmpconfDir, new ArrayList<>())) { + File oldFile = new File(confDir, file.getPath().substring(tmpconfDir.getPath().length(), file.getPath().length())); + if (!oldFile.getParentFile().exists()) { + status = oldFile.getParentFile().mkdirs(); + if (!status) { + throw new SolrException(ErrorCode.SERVER_ERROR, + "Unable to mkdirs: " + oldFile.getParentFile()); + } + } + if (oldFile.exists()) { + File backupFile = new File(oldFile.getPath() + "." + getDateAsStr(new Date(oldFile.lastModified()))); + if (!backupFile.getParentFile().exists()) { + status = backupFile.getParentFile().mkdirs(); + if (!status) { + throw new SolrException(ErrorCode.SERVER_ERROR, + "Unable to mkdirs: " + backupFile.getParentFile()); + } + } + status = oldFile.renameTo(backupFile); + if (!status) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unable to rename: " + oldFile + " to: " + backupFile); + } + } + status = file.renameTo(oldFile); + if (!status) { + throw new SolrException(ErrorCode.SERVER_ERROR, + "Unable to rename: " + file + " to: " + oldFile); + } + } + } + + /** + * The tlog files are moved from the tmp dir to the tlog dir as an atomic filesystem operation. + * A backup of the old directory is maintained. If the directory move fails, it will try to revert back the original + * tlog directory. + */ + private boolean copyTmpTlogFiles2Tlog(File tmpTlogDir) { + Path tlogDir = FileSystems.getDefault().getPath(solrCore.getUpdateHandler().getUpdateLog().getLogDir()); + Path backupTlogDir = FileSystems.getDefault().getPath(tlogDir.getParent().toAbsolutePath().toString(), tmpTlogDir.getName()); + + try { + Files.move(tlogDir, backupTlogDir, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + SolrException.log(LOG, "Unable to rename: " + tlogDir + " to: " + backupTlogDir, e); + return false; + } + + Path src = FileSystems.getDefault().getPath(backupTlogDir.toAbsolutePath().toString(), tmpTlogDir.getName()); + try { + Files.move(src, tlogDir, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + SolrException.log(LOG, "Unable to rename: " + src + " to: " + tlogDir, e); + + // In case of error, try to revert back the original tlog directory + try { + Files.move(backupTlogDir, tlogDir, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e2) { + // bad, we were not able to revert back the original tlog directory + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unable to rename: " + backupTlogDir + " to: " + tlogDir); + } + + return false; + } + + return true; + } + + private String getDateAsStr(Date d) { + return new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(d); + } + + private final Map confFileInfoCache = new HashMap<>(); + + /** + * The local conf files are compared with the conf files in the master. If they are same (by checksum) do not copy. + * + * @param confFilesToDownload The list of files obtained from master + * + * @return a list of configuration files which have changed on the master and need to be downloaded. + */ + private Collection> getModifiedConfFiles(List> confFilesToDownload) { + if (confFilesToDownload == null || confFilesToDownload.isEmpty()) + return Collections.EMPTY_LIST; + //build a map with alias/name as the key + Map> nameVsFile = new HashMap<>(); + NamedList names = new NamedList(); + for (Map map : confFilesToDownload) { + //if alias is present that is the name the file may have in the slave + String name = (String) (map.get(ALIAS) == null ? map.get(NAME) : map.get(ALIAS)); + nameVsFile.put(name, map); + names.add(name, null); + } + //get the details of the local conf files with the same alias/name + List> localFilesInfo = replicationHandler.getConfFileInfoFromCache(names, confFileInfoCache); + //compare their size/checksum to see if + for (Map fileInfo : localFilesInfo) { + String name = (String) fileInfo.get(NAME); + Map m = nameVsFile.get(name); + if (m == null) continue; // the file is not even present locally (so must be downloaded) + if (m.get(CHECKSUM).equals(fileInfo.get(CHECKSUM))) { + nameVsFile.remove(name); //checksums are same so the file need not be downloaded + } + } + return nameVsFile.isEmpty() ? Collections.EMPTY_LIST : nameVsFile.values(); + } + + /** + * This simulates File.delete exception-wise, since this class has some strange behavior with it. + * The only difference is it returns null on success, throws SecurityException on SecurityException, + * otherwise returns Throwable preventing deletion (instead of false), for additional information. + */ + static Throwable delete(File file) { + try { + Files.delete(file.toPath()); + return null; + } catch (SecurityException e) { + throw e; + } catch (Throwable other) { + return other; + } + } + + static boolean delTree(File dir) { + try { + org.apache.lucene.util.IOUtils.rm(dir.toPath()); + return true; + } catch (IOException e) { + LOG.warn("Unable to delete directory : " + dir, e); + return false; + } + } + + /** + * Stops the ongoing fetch + */ + void abortFetch() { + stop = true; + } + + @SuppressForbidden(reason = "Need currentTimeMillis for debugging/stats") + private void markReplicationStart() { + replicationTimer = new RTimer(); + replicationStartTimeStamp = new Date(); + } + + private void markReplicationStop() { + replicationStartTimeStamp = null; + replicationTimer = null; + } + + Date getReplicationStartTimeStamp() { + return replicationStartTimeStamp; + } + + long getReplicationTimeElapsed() { + long timeElapsed = 0; + if (replicationStartTimeStamp != null) + timeElapsed = TimeUnit.SECONDS.convert((long) replicationTimer.getTime(), TimeUnit.MILLISECONDS); + return timeElapsed; + } + + List> getTlogFilesToDownload() { + //make a copy first because it can be null later + List> tmp = tlogFilesToDownload; + //create a new instance. or else iterator may fail + return tmp == null ? Collections.EMPTY_LIST : new ArrayList<>(tmp); + } + + List> getTlogFilesDownloaded() { + //make a copy first because it can be null later + List> tmp = tlogFilesDownloaded; + // NOTE: it's safe to make a copy of a SynchronizedCollection(ArrayList) + return tmp == null ? Collections.EMPTY_LIST : new ArrayList<>(tmp); + } + + List> getConfFilesToDownload() { + //make a copy first because it can be null later + List> tmp = confFilesToDownload; + //create a new instance. or else iterator may fail + return tmp == null ? Collections.EMPTY_LIST : new ArrayList<>(tmp); + } + + List> getConfFilesDownloaded() { + //make a copy first because it can be null later + List> tmp = confFilesDownloaded; + // NOTE: it's safe to make a copy of a SynchronizedCollection(ArrayList) + return tmp == null ? Collections.EMPTY_LIST : new ArrayList<>(tmp); + } + + List> getFilesToDownload() { + //make a copy first because it can be null later + List> tmp = filesToDownload; + return tmp == null ? Collections.EMPTY_LIST : new ArrayList<>(tmp); + } + + List> getFilesDownloaded() { + List> tmp = filesDownloaded; + return tmp == null ? Collections.EMPTY_LIST : new ArrayList<>(tmp); + } + + // TODO: currently does not reflect conf files + Map getCurrentFile() { + Map tmp = currentFile; + DirectoryFileFetcher tmpFileFetcher = dirFileFetcher; + if (tmp == null) + return null; + tmp = new HashMap<>(tmp); + if (tmpFileFetcher != null) + tmp.put("bytesDownloaded", tmpFileFetcher.getBytesDownloaded()); + return tmp; + } + + private static class ReplicationHandlerException extends InterruptedException { + public ReplicationHandlerException(String message) { + super(message); + } + } + + private interface FileInterface { + public void sync() throws IOException; + public void write(byte[] buf, int packetSize) throws IOException; + public void close() throws Exception; + public void delete() throws Exception; + } + + /** + * The class acts as a client for ReplicationHandler.FileStream. It understands the protocol of wt=filestream + */ + public class FileFetcher { + protected final FileInterface file; + protected boolean includeChecksum = false; + protected final String fileName; + protected final String saveAs; + protected final String solrParamOutput; + protected final Long indexGen; + + protected final long size; + protected long bytesDownloaded = 0; + protected byte[] buf = new byte[1024 * 1024]; + protected final Checksum checksum; + protected int errorCount = 0; + protected boolean aborted = false; + + FileFetcher(FileInterface file, Map fileDetails, String saveAs, + String solrParamOutput, long latestGen) throws IOException { + this.file = file; + + if (fileDetails != null) { + this.fileName = (String) fileDetails.get(NAME); + this.size = (Long) fileDetails.get(SIZE); + } + else { + this.fileName = "somename"; + this.size = 0; + } + + this.solrParamOutput = solrParamOutput; + this.saveAs = saveAs; + indexGen = latestGen; + if (includeChecksum) { + checksum = new Adler32(); + } else { + checksum = null; + } + } + + public long getBytesDownloaded() { + return bytesDownloaded; + } + + /** + * The main method which downloads file + */ + public void fetchFile() throws Exception { + bytesDownloaded = 0; + try { + fetch(); + } catch(Exception e) { + if (!aborted) { + SolrException.log(IndexFetcher.LOG, "Error fetching file, doing one retry...", e); + // one retry + fetch(); + } else { + throw e; + } + } + } + + protected void fetch() throws Exception { + try { + while (true) { + final FastInputStream is = getStream(); + int result; + try { + //fetch packets one by one in a single request + result = fetchPackets(is); + if (result == 0 || result == NO_CONTENT) { + + return; + } + //if there is an error continue. But continue from the point where it got broken + } finally { + IOUtils.closeQuietly(is); + } + } + } finally { + cleanup(); + //if cleanup succeeds . The file is downloaded fully. do an fsync + fsyncService.submit(() -> { + try { + file.sync(); + } catch (IOException e) { + fsyncException = e; + } + }); + } + } + + protected int fetchPackets(FastInputStream fis) throws Exception { + byte[] intbytes = new byte[4]; + byte[] longbytes = new byte[8]; + try { + + while (true) { + if (stop) { + stop = false; + aborted = true; + throw new ReplicationHandlerException("User aborted replication"); + } + long checkSumServer = -1; + fis.readFully(intbytes); + //read the size of the packet + int packetSize = readInt(intbytes); + if (packetSize <= 0) { + LOG.warn("No content received for file: {}", fileName); + return NO_CONTENT; + } + if (buf.length < packetSize) + buf = new byte[packetSize]; + if (checksum != null) { + //read the checksum + fis.readFully(longbytes); + checkSumServer = readLong(longbytes); + } + //then read the packet of bytes + fis.readFully(buf, 0, packetSize); + //compare the checksum as sent from the master + if (includeChecksum) { + checksum.reset(); + checksum.update(buf, 0, packetSize); + long checkSumClient = checksum.getValue(); + if (checkSumClient != checkSumServer) { + LOG.error("Checksum not matched between client and server for file: {}", fileName); + //if checksum is wrong it is a problem return for retry + return 1; + } + } + //if everything is fine, write down the packet to the file + file.write(buf, packetSize); + bytesDownloaded += packetSize; + LOG.debug("Fetched and wrote {} bytes of file: {}", bytesDownloaded, fileName); + if (bytesDownloaded >= size) + return 0; + //errorCount is always set to zero after a successful packet + errorCount = 0; + } + } catch (ReplicationHandlerException e) { + throw e; + } catch (Exception e) { + LOG.warn("Error in fetching file: {} (downloaded {} of {} bytes)", + new Object[]{ fileName, bytesDownloaded, size, e}); + //for any failure, increment the error count + errorCount++; + //if it fails for the same packet for MAX_RETRIES fail and come out + if (errorCount > MAX_RETRIES) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to fetch file: " + fileName + + " (downloaded " + bytesDownloaded + " of " + size + " bytes" + + ", error count: " + errorCount + " > " + MAX_RETRIES + ")", e); + } + return ERR; + } + } + + /** + * The webcontainer flushes the data only after it fills the buffer size. So, all data has to be read as readFully() + * other wise it fails. So read everything as bytes and then extract an integer out of it + */ + protected int readInt(byte[] b) { + return (((b[0] & 0xff) << 24) | ((b[1] & 0xff) << 16) + | ((b[2] & 0xff) << 8) | (b[3] & 0xff)); + + } + + /** + * Same as above but to read longs from a byte array + */ + protected long readLong(byte[] b) { + return (((long) (b[0] & 0xff)) << 56) | (((long) (b[1] & 0xff)) << 48) + | (((long) (b[2] & 0xff)) << 40) | (((long) (b[3] & 0xff)) << 32) + | (((long) (b[4] & 0xff)) << 24) | ((b[5] & 0xff) << 16) + | ((b[6] & 0xff) << 8) | ((b[7] & 0xff)); + + } + + /** + * cleanup everything + */ + private void cleanup() { + try { + file.close(); + } catch (Exception e) {/* no-op */ + LOG.error("Error closing file: {}", this.saveAs, e); + } + if (bytesDownloaded != size) { + //if the download is not complete then + //delete the file being downloaded + try { + file.delete(); + } catch (Exception e) { + LOG.error("Error deleting file: {}", this.saveAs, e); + } + //if the failure is due to a user abort it is returned normally else an exception is thrown + if (!aborted) + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unable to download " + fileName + " completely. Downloaded " + + bytesDownloaded + "!=" + size); + } + } + + /** + * Open a new stream using HttpClient + */ + protected FastInputStream getStream() throws IOException { + + ModifiableSolrParams params = new ModifiableSolrParams(); + +// //the method is command=filecontent + params.set(COMMAND, CMD_GET_FILE); + params.set(GENERATION, Long.toString(indexGen)); + params.set(CommonParams.QT, ReplicationHandler.PATH); + //add the version to download. This is used to reserve the download + params.set(solrParamOutput, fileName); + if (useInternalCompression) { + params.set(COMPRESSION, "true"); + } +// use checksum + if (this.includeChecksum) { + params.set(CHECKSUM, false); + } + //wt=filestream this is a custom protocol + params.set(CommonParams.WT, FILE_STREAM); + // This happen if there is a failure there is a retry. the offset= ensures that + // the server starts from the offset + if (bytesDownloaded > 0) { + params.set(OFFSET, Long.toString(bytesDownloaded)); + } + + + NamedList response; + InputStream is = null; + + // TODO use shardhandler + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl) + .withHttpClient(myHttpClient) + .withResponseParser(null) + .build() + ) { + client.setSoTimeout(60000); + client.setConnectionTimeout(15000); + QueryRequest req = new QueryRequest(params); + response = client.request(req); + is = (InputStream) response.get("stream"); + if(useInternalCompression) { + is = new InflaterInputStream(is); + } + return new FastInputStream(is); + } catch (Exception e) { + //close stream on error + org.apache.commons.io.IOUtils.closeQuietly(is); + throw new IOException("Could not download file '" + fileName + "'", e); + } + } + } + + private static class DirectoryFile implements FileInterface { + private final String saveAs; + private Directory copy2Dir; + private IndexOutput outStream; + + DirectoryFile(Directory tmpIndexDir, String saveAs) throws IOException { + this.saveAs = saveAs; + this.copy2Dir = tmpIndexDir; + outStream = copy2Dir.createOutput(this.saveAs, DirectoryFactory.IOCONTEXT_NO_CACHE); + } + + public void sync() throws IOException { + copy2Dir.sync(Collections.singleton(saveAs)); + } + + public void write(byte[] buf, int packetSize) throws IOException { + outStream.writeBytes(buf, 0, packetSize); + } + + public void close() throws Exception { + outStream.close(); + } + + public void delete() throws Exception { + copy2Dir.deleteFile(saveAs); + } + } + + private class DirectoryFileFetcher extends FileFetcher { + DirectoryFileFetcher(Directory tmpIndexDir, Map fileDetails, String saveAs, + String solrParamOutput, long latestGen) throws IOException { + super(new DirectoryFile(tmpIndexDir, saveAs), fileDetails, saveAs, solrParamOutput, latestGen); + } + } + + private static class LocalFsFile implements FileInterface { + private File copy2Dir; + + FileChannel fileChannel; + private FileOutputStream fileOutputStream; + File file; + + LocalFsFile(File dir, String saveAs) throws IOException { + this.copy2Dir = dir; + + this.file = new File(copy2Dir, saveAs); + + File parentDir = this.file.getParentFile(); + if( ! parentDir.exists() ){ + if ( ! parentDir.mkdirs() ) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to create (sub)directory for file: " + saveAs); + } + } + + this.fileOutputStream = new FileOutputStream(file); + this.fileChannel = this.fileOutputStream.getChannel(); + } + + public void sync() throws IOException { + FileUtils.sync(file); + } + + public void write(byte[] buf, int packetSize) throws IOException { + fileChannel.write(ByteBuffer.wrap(buf, 0, packetSize)); + } + + public void close() throws Exception { + //close the FileOutputStream (which also closes the Channel) + fileOutputStream.close(); + } + + public void delete() throws Exception { + Files.delete(file.toPath()); + } + } + + class LocalFsFileFetcher extends FileFetcher { + LocalFsFileFetcher(File dir, Map fileDetails, String saveAs, + String solrParamOutput, long latestGen) throws IOException { + super(new LocalFsFile(dir, saveAs), fileDetails, saveAs, solrParamOutput, latestGen); + } + } + + + class ContentStoreFetcher extends FileFetcher { + private final File dir; + private final List> filesToDownload; + + ContentStoreFetcher(File dir, + String solrParamOutput, + long latestGen, + List> filesDetails) throws IOException { + super(null, null, null, solrParamOutput, latestGen); + this.dir = dir; + this.filesToDownload = filesDetails; + } + + @Override + protected void fetch() throws Exception { + try { + while (true) { + final FastInputStream is = getStream(); + int result; + try { + //fetch packets one by one in a single request + result = fetchPackets(is); + if (result == 0 || result == NO_CONTENT) { + return; + } + //if there is an error continue. But continue from the point where it got broken + } finally { + IOUtils.closeQuietly(is); + } + } + } finally { +// cleanup(); + //if cleanup succeeds . The file is downloaded fully. do an fsync +// fsyncService.submit(() -> { +// try { +// file.sync(); +// } catch (IOException e) { +// fsyncException = e; +// } +// }); + } + } + + + @Override + protected FastInputStream getStream() throws IOException { + + ModifiableSolrParams params = new ModifiableSolrParams(); + + params.set(COMMAND, CMD_CONTENT_STORE_FILES); + params.set(GENERATION, Long.toString(indexGen)); + params.set(CommonParams.QT, ReplicationHandler.PATH); + + List l = new ArrayList<>(); + params.set(CONTENT_STORE_FILE_LIST, + filesToDownload.stream() + .map(f -> (String) f.get(NAME)) + .toArray(String[]::new)); + + //add the version to download. This is used to reserve the download +// params.set(solrParamOutput, fileName); + if (useInternalCompression) { + params.set(COMPRESSION, "true"); + } + //use checksum + if (this.includeChecksum) { + params.set(CHECKSUM, true); + } + //wt=filestream this is a custom protocol + params.set(CommonParams.WT, FILE_STREAM); + // This happen if there is a failure there is a retry. the offset= ensures that + // the server starts from the offset + if (bytesDownloaded > 0) { + params.set(OFFSET, Long.toString(bytesDownloaded)); + } + + + NamedList response; + InputStream is = null; + + // TODO use shardhandler + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl) + .withHttpClient(myHttpClient) + .withResponseParser(null) + .build() + ) { + client.setSoTimeout(60000); + client.setConnectionTimeout(15000); + QueryRequest req = new QueryRequest(params); + response = client.request(req); + is = (InputStream) response.get("stream"); + if(useInternalCompression) { + is = new InflaterInputStream(is); + } + return new FastInputStream(is); + } catch (Exception e) { + //close stream on error + org.apache.commons.io.IOUtils.closeQuietly(is); + throw new IOException("Could not download file '" + fileName + "'", e); + } + } + + + // FIXME fetchpacket for stream of multiple files is not using checksum now. + @Override + protected int fetchPackets(FastInputStream fis) throws Exception { + byte[] intbytes = new byte[4]; + byte[] longbytes = new byte[8]; + try { + + while(true) { + fis.readFully(intbytes); + int fileNameSize = readInt(intbytes); + + byte[] filenameBytes = new byte[fileNameSize]; + fis.readFully(filenameBytes, 0, fileNameSize); + + String fileName = new String(filenameBytes); + + FileInterface file = new LocalFsFile(dir, fileName); + fis.readFully(intbytes); + + int fileSize = readInt(intbytes); + + + long fileSizeDownloaded = 0; + if (fileSize == 0){ + file.close(); + return 0; + } + + while (true) { + if (stop) { + stop = false; + aborted = true; + throw new ReplicationHandlerException("User aborted replication"); + } +// long checkSumServer = -1; +// fis.readFully(intbytes); + //read the size of the packet + int packetSize = (int) Math.min(PACKET_SZ, fileSize - fileSizeDownloaded); + if (packetSize <= 0) { + LOG.warn("No content received for file: {}", this.fileName); + file.close(); + return NO_CONTENT; + } + + if (buf.length < packetSize) + buf = new byte[packetSize]; +// if (checksum != null) { +// //read the checksum +// fis.readFully(longbytes); +// checkSumServer = readLong(longbytes); +// } + //then read the packet of bytes + fis.readFully(buf, 0, packetSize); + //compare the checksum as sent from the master +// if (includeChecksum) { +// checksum.reset(); +// checksum.update(buf, 0, packetSize); +// long checkSumClient = checksum.getValue(); +// if (checkSumClient != checkSumServer) { +// LOG.error("Checksum not matched between client and server for file: {}", this.fileName); +// //if checksum is wrong it is a problem return for retry +// return 1; +// } +// } + //if everything is fine, write down the packet to the file + file.write(buf, packetSize); + fileSizeDownloaded += packetSize; + LOG.warn("Fetched and wrote {} bytes of file: {}", fileSizeDownloaded, this.fileName); + if (fileSizeDownloaded >= fileSize) { + file.close(); + break; + } + //errorCount is always set to zero after a successful packet + errorCount = 0; + } + + bytesDownloaded += fileSizeDownloaded; + + } + } catch (ReplicationHandlerException e) { + throw e; + } catch (Exception e) { +// LOG.warn("Error in fetching file: {} (downloaded {} of {} bytes)", +// new Object[]{ fileName, bytesDownloaded, size, e}); +// //for any failure, increment the error count +// errorCount++; +// //if it fails for the same packet for MAX_RETRIES fail and come out +// if (errorCount > MAX_RETRIES) { +// throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, +// "Failed to fetch file: " + fileName + +// " (downloaded " + bytesDownloaded + " of " + size + " bytes" + +// ", error count: " + errorCount + " > " + MAX_RETRIES + ")", e); +// } +// return ERR; + return 0; + } + } + } + + NamedList getDetails() throws IOException, SolrServerException { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(COMMAND, CMD_DETAILS); + params.set("slave", false); + params.set(CommonParams.QT, ReplicationHandler.PATH); + + // TODO use shardhandler + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) { + client.setSoTimeout(60000); + client.setConnectionTimeout(15000); + QueryRequest request = new QueryRequest(params); + return client.request(request); + } + } + + public void destroy() { + abortFetch(); + } + + String getMasterUrl() { + return masterUrl; + } + + // TODO: ADDITIONAL METHOD + LocalFsFileFetcher newFileFetcher(File dir, Map fileDetails, String saveAs, + String solrParamOutput, long latestGen) throws IOException { + return new LocalFsFileFetcher(dir, fileDetails, saveAs, CONTENT_STORE_FILE, latestGen); + } + + private static final int MAX_RETRIES = 5; + + private static final int NO_CONTENT = 1; + + private static final int ERR = 2; + + public static final String REPLICATION_PROPERTIES = "replication.properties"; + + static final String INDEX_REPLICATED_AT = "indexReplicatedAt"; + + static final String TIMES_INDEX_REPLICATED = "timesIndexReplicated"; + + static final String CONF_FILES_REPLICATED = "confFilesReplicated"; + + static final String CONF_FILES_REPLICATED_AT = "confFilesReplicatedAt"; + + static final String TIMES_CONFIG_REPLICATED = "timesConfigReplicated"; + + static final String LAST_CYCLE_BYTES_DOWNLOADED = "lastCycleBytesDownloaded"; + + static final String TIMES_FAILED = "timesFailed"; + + static final String REPLICATION_FAILED_AT = "replicationFailedAt"; + + static final String PREVIOUS_CYCLE_TIME_TAKEN = "previousCycleTimeInSeconds"; + + static final String INDEX_REPLICATED_AT_LIST = "indexReplicatedAtList"; + + static final String REPLICATION_FAILED_AT_LIST = "replicationFailedAtList"; +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java new file mode 100644 index 000000000..a8da64a57 --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java @@ -0,0 +1,53 @@ +package org.alfresco.solr.handler; + +import java.net.URI; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +class OldBackupDirectory implements Comparable { + private static final Pattern dirNamePattern = Pattern.compile("^snapshot[.](.*)$"); + + private URI basePath; + private String dirName; + private Optional timestamp = Optional.empty(); + + public OldBackupDirectory(URI basePath, String dirName) { + this.dirName = Objects.requireNonNull(dirName); + this.basePath = Objects.requireNonNull(basePath); + Matcher m = dirNamePattern.matcher(dirName); + if (m.find()) { + try { + this.timestamp = Optional.of(new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).parse(m.group(1))); + } catch (ParseException e) { + this.timestamp = Optional.empty(); + } + } + } + + public URI getPath() { + return this.basePath.resolve(dirName); + } + + public String getDirName() { + return dirName; + } + + public Optional getTimestamp() { + return timestamp; + } + + @Override + public int compareTo(OldBackupDirectory that) { + if(this.timestamp.isPresent() && that.timestamp.isPresent()) { + return that.timestamp.get().compareTo(this.timestamp.get()); + } + // Use absolute value of path in case the time-stamp is missing on either side. + return that.getPath().compareTo(this.getPath()); + } +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java new file mode 100644 index 000000000..2825c7041 --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java @@ -0,0 +1,1934 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.alfresco.solr.handler; + +import org.alfresco.solr.content.ContentStoreCache; +import org.apache.commons.io.IOUtils; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexDeletionPolicy; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.RateLimiter; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.FastOutputStream; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.common.util.SuppressForbidden; +import org.apache.solr.core.CloseHook; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.DirectoryFactory.DirContext; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrDeletionPolicy; +import org.apache.solr.core.SolrEventListener; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.core.backup.repository.LocalFileSystemRepository; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.handler.RestoreCore; +import org.apache.solr.handler.SnapShooter; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.CdcrUpdateLog; +import org.apache.solr.update.SolrIndexWriter; +import org.apache.solr.update.VersionInfo; +import org.apache.solr.util.DefaultSolrThreadFactory; +import org.apache.solr.util.NumberUtils; +import org.apache.solr.util.PropertiesInputStream; +import org.apache.solr.util.RefCounted; +import org.apache.solr.util.plugin.SolrCoreAware; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.Adler32; +import java.util.zip.Checksum; +import java.util.zip.DeflaterOutputStream; + +import static java.util.Optional.ofNullable; +import static org.apache.solr.common.params.CommonParams.NAME; + +/** + *

A Handler which provides a REST API for replication and serves replication requests from Slaves.

+ *

When running on the master, it provides the following commands

  1. Get the current replicable index version + * (command=indexversion)
  2. Get the list of files for a given index version + * (command=filelist&indexversion=<VERSION>)
  3. Get full or a part (chunk) of a given index or a config + * file (command=filecontent&file=<FILE_NAME>) You can optionally specify an offset and length to get that + * chunk of the file. You can request a configuration file by using "cf" parameter instead of the "file" parameter.
    4. + *
  4. Get status/statistics (command=details)

When running on the slave, it provides the following + * commands

  1. Perform an index fetch now (command=snappull)
  2. Get status/statistics (command=details)
    3. + *
  3. Abort an index fetch (command=abort)
  4. Enable/Disable polling the master for new versions (command=enablepoll + * or command=disablepoll)
+ * + * + * @since solr 1.4 + */ +public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAware { + + public static final String PATH = "/replication"; + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + SolrCore core; + + private volatile boolean closed = false; + + private static final class CommitVersionInfo { + public final long version; + public final long generation; + private CommitVersionInfo(long g, long v) { + generation = g; + version = v; + } + /** + * builds a CommitVersionInfo data for the specified IndexCommit. + * Will never be null, ut version and generation may be zero if + * there are problems extracting them from the commit data + */ + public static CommitVersionInfo build(IndexCommit commit) { + long generation = commit.getGeneration(); + long version = 0; + try { + final Map commitData = commit.getUserData(); + String commitTime = commitData.get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY); + if (commitTime != null) { + try { + version = Long.parseLong(commitTime); + } catch (NumberFormatException e) { + LOG.warn("Version in commitData was not formatted correctly: " + commitTime, e); + } + } + } catch (IOException e) { + LOG.warn("Unable to get version from commitData, commit: " + commit, e); + } + return new CommitVersionInfo(generation, version); + } + } + + private IndexFetcher pollingIndexFetcher; + + private ReentrantLock indexFetchLock = new ReentrantLock(); + + private ExecutorService restoreExecutor = ExecutorUtil.newMDCAwareSingleThreadExecutor( + new DefaultSolrThreadFactory("restoreExecutor")); + + private volatile Future restoreFuture; + + private volatile String currentRestoreName; + + private String includeConfFiles; + + private NamedList confFileNameAlias = new NamedList<>(); + + private boolean isMaster = false; + + private boolean isSlave = false; + + private boolean replicateOnOptimize = false; + + private boolean replicateOnCommit = false; + + private boolean replicateOnStart = false; + + private ScheduledExecutorService executorService; + + private volatile long executorStartTime; + + private int numberBackupsToKeep = 0; //zero: do not delete old backups + + private int numTimesReplicated = 0; + + private final Map confFileInfoCache = new HashMap<>(); + + private Integer reserveCommitDuration = readIntervalMs("00:00:10"); + + volatile IndexCommit indexCommitPoint; + + volatile NamedList snapShootDetails; + + private AtomicBoolean replicationEnabled = new AtomicBoolean(true); + + private Long pollIntervalNs; + private String pollIntervalStr; + + /** + * Disable the timer task for polling + */ + private AtomicBoolean pollDisabled = new AtomicBoolean(false); + + String getPollInterval() { + return pollIntervalStr; + } + + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + rsp.setHttpCaching(false); + final SolrParams solrParams = req.getParams(); + String command = solrParams.get(COMMAND); + if (command == null) { + rsp.add(STATUS, OK_STATUS); + rsp.add("message", "No command"); + return; + } + // This command does not give the current index version of the master + // It gives the current 'replicateable' index version + if (command.equals(CMD_INDEX_VERSION)) { + IndexCommit commitPoint = indexCommitPoint; // make a copy so it won't change + + if (commitPoint == null) { + // if this handler is 'lazy', we may not have tracked the last commit + // because our commit listener is registered on inform + commitPoint = core.getDeletionPolicy().getLatestCommit(); + } + + if (commitPoint != null && replicationEnabled.get()) { + // + // There is a race condition here. The commit point may be changed / deleted by the time + // we get around to reserving it. This is a very small window though, and should not result + // in a catastrophic failure, but will result in the client getting an empty file list for + // the CMD_GET_FILE_LIST command. + // + core.getDeletionPolicy().setReserveDuration(commitPoint.getGeneration(), reserveCommitDuration); + rsp.add(CMD_INDEX_VERSION, IndexDeletionPolicyWrapper.getCommitTimestamp(commitPoint)); + rsp.add(GENERATION, commitPoint.getGeneration()); + } else { + // This happens when replication is not configured to happen after startup and no commit/optimize + // has happened yet. + rsp.add(CMD_INDEX_VERSION, 0L); + rsp.add(GENERATION, 0L); + } + } else if (command.equals(CMD_GET_FILE)) { + getFileStream(solrParams, rsp); + } else if (command.equals(CMD_CONTENT_STORE_FILES)) { + getContetntStore(new ModifiableSolrParams(solrParams), rsp); + }else if (command.equals(CMD_GET_FILE_LIST)) { + getFileList(solrParams, rsp); + } else if (command.equalsIgnoreCase(CMD_BACKUP)) { + doSnapShoot(new ModifiableSolrParams(solrParams), rsp, req); + rsp.add(STATUS, OK_STATUS); + } else if (command.equalsIgnoreCase(CMD_RESTORE)) { + restore(new ModifiableSolrParams(solrParams), rsp, req); + rsp.add(STATUS, OK_STATUS); + } else if (command.equalsIgnoreCase(CMD_RESTORE_STATUS)) { + rsp.add(CMD_RESTORE_STATUS, getRestoreStatus()); + } else if (command.equalsIgnoreCase(CMD_DELETE_BACKUP)) { + deleteSnapshot(new ModifiableSolrParams(solrParams)); + rsp.add(STATUS, OK_STATUS); + } else if (command.equalsIgnoreCase(CMD_FETCH_INDEX)) { + String masterUrl = solrParams.get(MASTER_URL); + if (!isSlave && masterUrl == null) { + rsp.add(STATUS,ERR_STATUS); + rsp.add("message","No slave configured or no 'masterUrl' Specified"); + return; + } + final SolrParams paramsCopy = new ModifiableSolrParams(solrParams); + Thread fetchThread = new Thread(() -> doFetch(paramsCopy, false), "explicit-fetchindex-cmd") ; + fetchThread.setDaemon(false); + fetchThread.start(); + if (solrParams.getBool(WAIT, false)) { + fetchThread.join(); + } + rsp.add(STATUS, OK_STATUS); + } else if (command.equalsIgnoreCase(CMD_DISABLE_POLL)) { + if (pollingIndexFetcher != null){ + disablePoll(); + rsp.add(STATUS, OK_STATUS); + } else { + rsp.add(STATUS, ERR_STATUS); + rsp.add("message","No slave configured"); + } + } else if (command.equalsIgnoreCase(CMD_ENABLE_POLL)) { + if (pollingIndexFetcher != null){ + enablePoll(); + rsp.add(STATUS, OK_STATUS); + }else { + rsp.add(STATUS,ERR_STATUS); + rsp.add("message","No slave configured"); + } + } else if (command.equalsIgnoreCase(CMD_ABORT_FETCH)) { + if (abortFetch()){ + rsp.add(STATUS, OK_STATUS); + } else { + rsp.add(STATUS,ERR_STATUS); + rsp.add("message","No slave configured"); + } + } else if (command.equals(CMD_SHOW_COMMITS)) { + rsp.add(CMD_SHOW_COMMITS, getCommits()); + } else if (command.equals(CMD_DETAILS)) { + rsp.add(CMD_DETAILS, getReplicationDetails(solrParams.getBool("slave", true))); + } else if (CMD_ENABLE_REPL.equalsIgnoreCase(command)) { + replicationEnabled.set(true); + rsp.add(STATUS, OK_STATUS); + } else if (CMD_DISABLE_REPL.equalsIgnoreCase(command)) { + replicationEnabled.set(false); + rsp.add(STATUS, OK_STATUS); + } + } + + public boolean abortFetch() { + IndexFetcher fetcher = currentIndexFetcher; + if (fetcher != null){ + fetcher.abortFetch(); + return true; + } else { + return false; + } + } + + // TODO: if we extend the default RH we could remove this method (and the snapshooter dep as well) + private void deleteSnapshot(ModifiableSolrParams params) { +// String name = params.get(NAME); +// if(name == null) { +// throw new SolrException(ErrorCode.BAD_REQUEST, "Missing mandatory param: name"); +// } +// +// SnapShooter snapShooter = new SnapShooter(core, params.get(CoreAdminParams.BACKUP_LOCATION), params.get(NAME)); +// snapShooter.validateDeleteSnapshot(); +// snapShooter.deleteSnapAsync(this); + } + + private List> getCommits() { + Map commits = core.getDeletionPolicy().getCommits(); + List> l = new ArrayList<>(); + + for (IndexCommit c : commits.values()) { + try { + NamedList nl = new NamedList<>(); + nl.add("indexVersion", IndexDeletionPolicyWrapper.getCommitTimestamp(c)); + nl.add(GENERATION, c.getGeneration()); + List commitList = new ArrayList<>(c.getFileNames().size()); + commitList.addAll(c.getFileNames()); + Collections.sort(commitList); + nl.add(CMD_GET_FILE_LIST, commitList); + l.add(nl); + } catch (IOException e) { + LOG.warn("Exception while reading files for commit " + c, e); + } + } + return l; + } + + static Long getCheckSum(Checksum checksum, File f) { + FileInputStream fis = null; + checksum.reset(); + byte[] buffer = new byte[1024 * 1024]; + int bytesRead; + try { + fis = new FileInputStream(f); + while ((bytesRead = fis.read(buffer)) >= 0) + checksum.update(buffer, 0, bytesRead); + return checksum.getValue(); + } catch (Exception e) { + LOG.warn("Exception in finding checksum of " + f, e); + } finally { + IOUtils.closeQuietly(fis); + } + return null; + } + + private volatile IndexFetcher currentIndexFetcher; + + public IndexFetcher.IndexFetchResult doFetch(SolrParams solrParams, boolean forceReplication) { + String masterUrl = solrParams == null ? null : solrParams.get(MASTER_URL); + if (!indexFetchLock.tryLock()) + return IndexFetcher.IndexFetchResult.LOCK_OBTAIN_FAILED; + try { + if (masterUrl != null) { + if (currentIndexFetcher != null && currentIndexFetcher != pollingIndexFetcher) { + currentIndexFetcher.destroy(); + } + currentIndexFetcher = new IndexFetcher(solrParams.toNamedList(), this, core); + } else { + currentIndexFetcher = pollingIndexFetcher; + } + return currentIndexFetcher.fetchLatestIndex(forceReplication); + } catch (Exception e) { + SolrException.log(LOG, "Index fetch failed ", e); + return new IndexFetcher.IndexFetchResult(IndexFetcher.IndexFetchResult.FAILED_BY_EXCEPTION_MESSAGE, false, e); + } finally { + if (pollingIndexFetcher != null) { + currentIndexFetcher = pollingIndexFetcher; + } + indexFetchLock.unlock(); + } + } + + boolean isReplicating() { + return indexFetchLock.isLocked(); + } + + private void restore(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) throws IOException { + if (restoreFuture != null && !restoreFuture.isDone()) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Restore in progress. Cannot run multiple restore operations" + + "for the same core"); + } + String name = params.get(NAME); + String location = params.get(CoreAdminParams.BACKUP_LOCATION); + + String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY); + CoreContainer cc = core.getCoreContainer(); + BackupRepository repo = null; + if (repoName != null) { + repo = cc.newBackupRepository(Optional.of(repoName)); + location = repo.getBackupLocation(location); + if (location == null) { + throw new IllegalArgumentException("location is required"); + } + } else { + repo = new LocalFileSystemRepository(); + } + + //If location is not provided then assume that the restore index is present inside the data directory. + if (location == null) { + location = core.getDataDir(); + } + + URI locationUri = repo.createURI(location); + + //If name is not provided then look for the last unnamed( the ones with the snapshot.timestamp format) + //snapshot folder since we allow snapshots to be taken without providing a name. Pick the latest timestamp. + if (name == null) { + String[] filePaths = repo.listAll(locationUri); + List dirs = new ArrayList<>(); + for (String f : filePaths) { + OldBackupDirectory obd = new OldBackupDirectory(locationUri, f); + if (obd.getTimestamp().isPresent()) { + dirs.add(obd); + } + } + Collections.sort(dirs); + if (dirs.size() == 0) { + throw new SolrException(ErrorCode.BAD_REQUEST, "No backup name specified and none found in " + core.getDataDir()); + } + name = dirs.get(0).getDirName(); + } else { + //"snapshot." is prefixed by snapshooter + name = "snapshot." + name; + } + + RestoreCore restoreCore = new RestoreCore(repo, core, locationUri, name); + try { + MDC.put("RestoreCore.core", core.getName()); + MDC.put("RestoreCore.backupLocation", location); + MDC.put("RestoreCore.backupName", name); + restoreFuture = restoreExecutor.submit(restoreCore); + currentRestoreName = name; + } finally { + MDC.remove("RestoreCore.core"); + MDC.remove("RestoreCore.backupLocation"); + MDC.remove("RestoreCore.backupName"); + } + } + + private NamedList getRestoreStatus() { + NamedList status = new SimpleOrderedMap<>(); + + if (restoreFuture == null) { + status.add(STATUS, "No restore actions in progress"); + return status; + } + + status.add("snapshotName", currentRestoreName); + if (restoreFuture.isDone()) { + try { + boolean success = restoreFuture.get(); + if (success) { + status.add(STATUS, SUCCESS); + } else { + status.add(STATUS, FAILED); + } + } catch (Exception e) { + status.add(STATUS, FAILED); + status.add(EXCEPTION, e.getMessage()); + } + } else { + status.add(STATUS, "In Progress"); + } + return status; + } + + private void doSnapShoot(SolrParams params, SolrQueryResponse rsp, + SolrQueryRequest req) { + try { + int numberToKeep = params.getInt(NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM, 0); + if (numberToKeep > 0 && numberBackupsToKeep > 0) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Cannot use " + + NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM + " if " + + NUMBER_BACKUPS_TO_KEEP_INIT_PARAM + + " was specified in the configuration."); + } + numberToKeep = Math.max(numberToKeep, numberBackupsToKeep); + if (numberToKeep < 1) { + numberToKeep = Integer.MAX_VALUE; + } + + IndexCommit indexCommit = null; + String commitName = params.get(CoreAdminParams.COMMIT_NAME); + if (commitName != null) { + SolrSnapshotMetaDataManager snapshotMgr = core.getSnapshotMetaDataManager(); + Optional commit = snapshotMgr.getIndexCommitByName(commitName); + if(commit.isPresent()) { + indexCommit = commit.get(); + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an index commit with name " + commitName + + " for core " + core.getName()); + } + } else { + IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy(); + indexCommit = delPolicy.getLatestCommit(); + + if (indexCommit == null) { + indexCommit = req.getSearcher().getIndexReader().getIndexCommit(); + } + } + + String location = params.get(CoreAdminParams.BACKUP_LOCATION); + String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY); + CoreContainer cc = core.getCoreContainer(); + BackupRepository repo = null; + if (repoName != null) { + repo = cc.newBackupRepository(Optional.of(repoName)); + location = repo.getBackupLocation(location); + if (location == null) { + throw new IllegalArgumentException("location is required"); + } + } else { + repo = new LocalFileSystemRepository(); + if (location == null) { + location = core.getDataDir(); + } else { + location = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString(); + } + } + + // small race here before the commit point is saved + URI locationUri = repo.createURI(location); + SnapShooter snapShooter = new SnapShooter(repo, core, locationUri, params.get(NAME), commitName); + snapShooter.validateCreateSnapshot(); + snapShooter.createSnapAsync(indexCommit, numberToKeep, (nl) -> snapShootDetails = nl); + + } catch (Exception e) { + LOG.warn("Exception during creating a snapshot", e); + rsp.add("exception", e); + } + } + + /** + * This method adds an Object of FileStream to the response . The FileStream implements a custom protocol which is + * understood by IndexFetcher.FileFetcher + */ + // TODO: being private, this method must be changed (for including the CS files management) + private void getFileStream(SolrParams solrParams, SolrQueryResponse rsp) { + ModifiableSolrParams rawParams = new ModifiableSolrParams(solrParams); + rawParams.set(CommonParams.WT, FILE_STREAM); + + String csFileName = solrParams.get(CONTENT_STORE_FILE); + String cfileName = solrParams.get(CONF_FILE_SHORT); + String tlogFileName = solrParams.get(TLOG_FILE); + String contentStoreFileName = solrParams.get(CONTENT_STORE_FILE); + if (cfileName != null) { + rsp.add(FILE_STREAM, new LocalFsConfFileStream(solrParams)); + } else if (tlogFileName != null) { + rsp.add(FILE_STREAM, new LocalFsTlogFileStream(solrParams)); + } + else if (contentStoreFileName != null) { + rsp.add(FILE_STREAM, new LocalContentStoreFileStream(solrParams)); + } + else { + rsp.add(FILE_STREAM, new DirectoryFileStream(solrParams)); + } + } + + + private void getContetntStore(SolrParams solrParams, SolrQueryResponse rsp){ + rsp.add(FILE_STREAM, new ContentStoreFilesStream(solrParams)); + } + + @SuppressWarnings("unchecked") + private void getFileList(SolrParams solrParams, SolrQueryResponse rsp) { + String v = solrParams.get(GENERATION); + if (v == null) { + rsp.add("status", "no index generation specified"); + return; + } + long gen = Long.parseLong(v); + IndexCommit commit = core.getDeletionPolicy().getCommitPoint(gen); + + //System.out.println("ask for files for gen:" + commit.getGeneration() + core.getCoreDescriptor().getCoreContainer().getZkController().getNodeName()); + if (commit == null) { + rsp.add("status", "invalid index generation"); + return; + } + + // reserve the indexcommit for sometime + core.getDeletionPolicy().setReserveDuration(gen, reserveCommitDuration); + List> result = new ArrayList<>(); + Directory dir = null; + try { + dir = core.getDirectoryFactory().get(core.getNewIndexDir(), DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); + SegmentInfos infos = SegmentInfos.readCommit(dir, commit.getSegmentsFileName()); + + for (SegmentCommitInfo commitInfo : infos) { + for (String file : commitInfo.files()) { + Map fileMeta = new HashMap<>(); + fileMeta.put(NAME, file); + fileMeta.put(SIZE, dir.fileLength(file)); + + try (final IndexInput in = dir.openInput(file, IOContext.READONCE)) { + try { + long checksum = CodecUtil.retrieveChecksum(in); + fileMeta.put(CHECKSUM, checksum); + } catch (Exception e) { + LOG.warn("Could not read checksum from index file: " + file, e); + } + } + + result.add(fileMeta); + } + } + + // add the segments_N file + + Map fileMeta = new HashMap<>(); + fileMeta.put(NAME, infos.getSegmentsFileName()); + fileMeta.put(SIZE, dir.fileLength(infos.getSegmentsFileName())); + if (infos.getId() != null) { + try (final IndexInput in = dir.openInput(infos.getSegmentsFileName(), IOContext.READONCE)) { + try { + fileMeta.put(CHECKSUM, CodecUtil.retrieveChecksum(in)); + } catch (Exception e) { + LOG.warn("Could not read checksum from index file: " + infos.getSegmentsFileName(), e); + } + } + } + result.add(fileMeta); + } catch (IOException e) { + rsp.add("status", "unable to get file names for given index generation"); + rsp.add(EXCEPTION, e); + LOG.error("Unable to get file names for indexCommit generation: " + gen, e); + } finally { + if (dir != null) { + try { + core.getDirectoryFactory().release(dir); + } catch (IOException e) { + SolrException.log(LOG, "Could not release directory after fetching file list", e); + } + } + } + rsp.add(CMD_GET_FILE_LIST, result); + + // fetch list of tlog files only if cdcr is activated + if (solrParams.getBool(TLOG_FILES, true) && core.getUpdateHandler().getUpdateLog() != null + && core.getUpdateHandler().getUpdateLog() instanceof CdcrUpdateLog) { + try { + List> tlogfiles = getTlogFileList(commit); + LOG.info("Adding tlog files to list: " + tlogfiles); + rsp.add(TLOG_FILES, tlogfiles); + } + catch (IOException e) { + rsp.add("status", "unable to get tlog file names for given index generation"); + rsp.add(EXCEPTION, e); + LOG.error("Unable to get tlog file names for indexCommit generation: " + gen, e); + } + } + + if (confFileNameAlias.size() < 1 || core.getCoreContainer().isZooKeeperAware()) + return; + LOG.debug("Adding config files to list: " + includeConfFiles); + //if configuration files need to be included get their details + rsp.add(CONF_FILES, getConfFileInfoFromCache(confFileNameAlias, confFileInfoCache)); + + rsp.add(CONTENT_STORE_FILES, getContentStoreFileList(commit)); + + } + + /** + * Retrieves the list of tlog files associated to a commit point. + */ + List> getTlogFileList(IndexCommit commit) throws IOException { + long maxVersion = this.getMaxVersion(commit); + CdcrUpdateLog ulog = (CdcrUpdateLog) core.getUpdateHandler().getUpdateLog(); + String[] logList = ulog.getLogList(new File(ulog.getLogDir())); + List> tlogFiles = new ArrayList<>(); + for (String fileName : logList) { + // filter out tlogs that are older than the current index commit generation, so that the list of tlog files is + // in synch with the latest index commit point + long startVersion = Math.abs(Long.parseLong(fileName.substring(fileName.lastIndexOf('.') + 1))); + if (startVersion < maxVersion) { + Map fileMeta = new HashMap<>(); + fileMeta.put(NAME, fileName); + fileMeta.put(SIZE, new File(ulog.getLogDir(), fileName).length()); + tlogFiles.add(fileMeta); + } + } + return tlogFiles; + } + + List> getContentStoreFileList(IndexCommit commit) { + + List> contentStoreFiles = new ArrayList<>(); + String contentStoreDirectory = ContentStoreCache.get().getContentStoreRootPath(); + + try { + Files.walk(Paths.get(contentStoreDirectory)).forEach(p -> { + String filename = p.toString().replaceFirst(contentStoreDirectory, ""); + File f = new File(p.toUri()); + if (!f.isDirectory()) { + Checksum checksum = new Adler32(); + FileInfo info = new FileInfo(f.lastModified(), filename, f.length(), getCheckSum(checksum, f)); + contentStoreFiles.add(info.getAsMap()); + } + }); + } catch (IOException e) { + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + } + + return contentStoreFiles; + } + + /** + * Retrieves the maximum version number from an index commit. + */ + private long getMaxVersion(IndexCommit commit) throws IOException { + try (DirectoryReader reader = DirectoryReader.open(commit)) { + IndexSearcher searcher = new IndexSearcher(reader); + VersionInfo vinfo = core.getUpdateHandler().getUpdateLog().getVersionInfo(); + return Math.abs(vinfo.getMaxVersionFromIndex(searcher)); + } + } + + /** + * For configuration files, checksum of the file is included because, unlike index files, they may have same content + * but different timestamps. + *

+ * The local conf files information is cached so that everytime it does not have to compute the checksum. The cache is + * refreshed only if the lastModified of the file changes + */ + List> getConfFileInfoFromCache(NamedList nameAndAlias, + final Map confFileInfoCache) { + List> confFiles = new ArrayList<>(); + synchronized (confFileInfoCache) { + File confDir = new File(core.getResourceLoader().getConfigDir()); + Checksum checksum = null; + for (int i = 0; i < nameAndAlias.size(); i++) { + String cf = nameAndAlias.getName(i); + File f = new File(confDir, cf); + if (!f.exists() || f.isDirectory()) continue; //must not happen + FileInfo info = confFileInfoCache.get(cf); + if (info == null || info.lastmodified != f.lastModified() || info.size != f.length()) { + if (checksum == null) checksum = new Adler32(); + info = new FileInfo(f.lastModified(), cf, f.length(), getCheckSum(checksum, f)); + confFileInfoCache.put(cf, info); + } + Map m = info.getAsMap(); + if (nameAndAlias.getVal(i) != null) m.put(ALIAS, nameAndAlias.getVal(i)); + confFiles.add(m); + } + } + return confFiles; + } + + + + static class FileInfo { + long lastmodified; + String name; + long size; + long checksum; + + public FileInfo(long lasmodified, String name, long size, long checksum) { + this.lastmodified = lasmodified; + this.name = name; + this.size = size; + this.checksum = checksum; + } + + Map getAsMap() { + Map map = new HashMap<>(); + map.put(NAME, name); + map.put(SIZE, size); + map.put(CHECKSUM, checksum); + return map; + } + } + + void disablePoll() { + if (isSlave) { + pollDisabled.set(true); + LOG.info("inside disable poll, value of pollDisabled = " + pollDisabled); + } + } + + void enablePoll() { + if (isSlave) { + pollDisabled.set(false); + LOG.info("inside enable poll, value of pollDisabled = " + pollDisabled); + } + } + + boolean isPollingDisabled() { + return pollDisabled.get(); + } + + @SuppressForbidden(reason = "Need currentTimeMillis, to output next execution time in replication details") + private void markScheduledExecutionStart() { + executorStartTime = System.currentTimeMillis(); + } + + private Date getNextScheduledExecTime() { + Date nextTime = null; + if (executorStartTime > 0) + nextTime = new Date(executorStartTime + TimeUnit.MILLISECONDS.convert(pollIntervalNs, TimeUnit.NANOSECONDS)); + return nextTime; + } + + int getTimesReplicatedSinceStartup() { + return numTimesReplicated; + } + + void setTimesReplicatedSinceStartup() { + numTimesReplicated++; + } + + @Override + public Category getCategory() { + return Category.REPLICATION; + } + + @Override + public String getDescription() { + return "ReplicationHandler provides replication of index and configuration files from Master to Slaves"; + } + + /** + * returns the CommitVersionInfo for the current searcher, or null on error. + */ + private CommitVersionInfo getIndexVersion() { + CommitVersionInfo v = null; + RefCounted searcher = core.getSearcher(); + try { + v = CommitVersionInfo.build(searcher.get().getIndexReader().getIndexCommit()); + } catch (IOException e) { + LOG.warn("Unable to get index commit: ", e); + } finally { + searcher.decref(); + } + return v; + } + + @Override + @SuppressWarnings("unchecked") + public NamedList getStatistics() { + NamedList list = super.getStatistics(); + if (core != null) { + list.add("indexSize", NumberUtils.readableSize(core.getIndexSize())); + CommitVersionInfo vInfo = (core != null && !core.isClosed()) ? getIndexVersion(): null; + list.add("indexVersion", null == vInfo ? 0 : vInfo.version); + list.add(GENERATION, null == vInfo ? 0 : vInfo.generation); + + list.add("indexPath", core.getIndexDir()); + list.add("isMaster", String.valueOf(isMaster)); + list.add("isSlave", String.valueOf(isSlave)); + + IndexFetcher fetcher = currentIndexFetcher; + if (fetcher != null) { + list.add(MASTER_URL, fetcher.getMasterUrl()); + if (getPollInterval() != null) { + list.add(POLL_INTERVAL, getPollInterval()); + } + list.add("isPollingDisabled", String.valueOf(isPollingDisabled())); + list.add("isReplicating", String.valueOf(isReplicating())); + long elapsed = fetcher.getReplicationTimeElapsed(); + long val = fetcher.getTotalBytesDownloaded(); + if (elapsed > 0) { + list.add("timeElapsed", elapsed); + list.add("bytesDownloaded", val); + list.add("downloadSpeed", val / elapsed); + } + Properties props = loadReplicationProperties(); + addVal(list, IndexFetcher.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class); + addVal(list, IndexFetcher.INDEX_REPLICATED_AT, props, Date.class); + addVal(list, IndexFetcher.CONF_FILES_REPLICATED_AT, props, Date.class); + addVal(list, IndexFetcher.REPLICATION_FAILED_AT, props, Date.class); + addVal(list, IndexFetcher.TIMES_FAILED, props, Integer.class); + addVal(list, IndexFetcher.TIMES_INDEX_REPLICATED, props, Integer.class); + addVal(list, IndexFetcher.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class); + addVal(list, IndexFetcher.TIMES_CONFIG_REPLICATED, props, Integer.class); + addVal(list, IndexFetcher.CONF_FILES_REPLICATED, props, String.class); + } + if (isMaster) { + if (includeConfFiles != null) list.add("confFilesToReplicate", includeConfFiles); + list.add(REPLICATE_AFTER, getReplicateAfterStrings()); + list.add("replicationEnabled", String.valueOf(replicationEnabled.get())); + } + } + return list; + } + + /** + * Used for showing statistics and progress information. + */ + private NamedList getReplicationDetails(boolean showSlaveDetails) { + NamedList details = new SimpleOrderedMap<>(); + NamedList master = new SimpleOrderedMap<>(); + NamedList slave = new SimpleOrderedMap<>(); + + details.add("indexSize", NumberUtils.readableSize(core.getIndexSize())); + details.add("indexPath", core.getIndexDir()); + details.add(CMD_SHOW_COMMITS, getCommits()); + details.add("isMaster", String.valueOf(isMaster)); + details.add("isSlave", String.valueOf(isSlave)); + CommitVersionInfo vInfo = getIndexVersion(); + details.add("indexVersion", null == vInfo ? 0 : vInfo.version); + details.add(GENERATION, null == vInfo ? 0 : vInfo.generation); + + IndexCommit commit = indexCommitPoint; // make a copy so it won't change + + if (isMaster) { + if (includeConfFiles != null) master.add(CONF_FILES, includeConfFiles); + master.add(REPLICATE_AFTER, getReplicateAfterStrings()); + master.add("replicationEnabled", String.valueOf(replicationEnabled.get())); + } + + if (isMaster && commit != null) { + CommitVersionInfo repCommitInfo = CommitVersionInfo.build(commit); + master.add("replicableVersion", repCommitInfo.version); + master.add("replicableGeneration", repCommitInfo.generation); + } + + IndexFetcher fetcher = currentIndexFetcher; + if (fetcher != null) { + Properties props = loadReplicationProperties(); + if (showSlaveDetails) { + try { + NamedList nl = fetcher.getDetails(); + slave.add("masterDetails", nl.get(CMD_DETAILS)); + } catch (Exception e) { + LOG.warn( + "Exception while invoking 'details' method for replication on master ", + e); + slave.add(ERR_STATUS, "invalid_master"); + } + } + slave.add(MASTER_URL, fetcher.getMasterUrl()); + if (getPollInterval() != null) { + slave.add(POLL_INTERVAL, getPollInterval()); + } + Date nextScheduled = getNextScheduledExecTime(); + if (nextScheduled != null && !isPollingDisabled()) { + slave.add(NEXT_EXECUTION_AT, nextScheduled.toString()); + } else if (isPollingDisabled()) { + slave.add(NEXT_EXECUTION_AT, "Polling disabled"); + } + addVal(slave, IndexFetcher.INDEX_REPLICATED_AT, props, Date.class); + addVal(slave, IndexFetcher.INDEX_REPLICATED_AT_LIST, props, List.class); + addVal(slave, IndexFetcher.REPLICATION_FAILED_AT_LIST, props, List.class); + addVal(slave, IndexFetcher.TIMES_INDEX_REPLICATED, props, Integer.class); + addVal(slave, IndexFetcher.CONF_FILES_REPLICATED, props, Integer.class); + addVal(slave, IndexFetcher.TIMES_CONFIG_REPLICATED, props, Integer.class); + addVal(slave, IndexFetcher.CONF_FILES_REPLICATED_AT, props, Integer.class); + addVal(slave, IndexFetcher.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class); + addVal(slave, IndexFetcher.TIMES_FAILED, props, Integer.class); + addVal(slave, IndexFetcher.REPLICATION_FAILED_AT, props, Date.class); + addVal(slave, IndexFetcher.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class); + + slave.add("currentDate", new Date().toString()); + slave.add("isPollingDisabled", String.valueOf(isPollingDisabled())); + boolean isReplicating = isReplicating(); + slave.add("isReplicating", String.valueOf(isReplicating)); + if (isReplicating) { + try { + long bytesToDownload = 0; + List filesToDownload = new ArrayList<>(); + for (Map file : fetcher.getFilesToDownload()) { + filesToDownload.add((String) file.get(NAME)); + bytesToDownload += (Long) file.get(SIZE); + } + + //get list of conf files to download + for (Map file : fetcher.getConfFilesToDownload()) { + filesToDownload.add((String) file.get(NAME)); + bytesToDownload += (Long) file.get(SIZE); + } + + slave.add("filesToDownload", filesToDownload); + slave.add("numFilesToDownload", String.valueOf(filesToDownload.size())); + slave.add("bytesToDownload", NumberUtils.readableSize(bytesToDownload)); + + long bytesDownloaded = 0; + List filesDownloaded = new ArrayList<>(); + for (Map file : fetcher.getFilesDownloaded()) { + filesDownloaded.add((String) file.get(NAME)); + bytesDownloaded += (Long) file.get(SIZE); + } + + //get list of conf files downloaded + for (Map file : fetcher.getConfFilesDownloaded()) { + filesDownloaded.add((String) file.get(NAME)); + bytesDownloaded += (Long) file.get(SIZE); + } + + Map currentFile = fetcher.getCurrentFile(); + String currFile = null; + long currFileSize = 0, currFileSizeDownloaded = 0; + float percentDownloaded = 0; + if (currentFile != null) { + currFile = (String) currentFile.get(NAME); + currFileSize = (Long) currentFile.get(SIZE); + if (currentFile.containsKey("bytesDownloaded")) { + currFileSizeDownloaded = (Long) currentFile.get("bytesDownloaded"); + bytesDownloaded += currFileSizeDownloaded; + if (currFileSize > 0) + percentDownloaded = (currFileSizeDownloaded * 100) / currFileSize; + } + } + slave.add("filesDownloaded", filesDownloaded); + slave.add("numFilesDownloaded", String.valueOf(filesDownloaded.size())); + + long estimatedTimeRemaining = 0; + + Date replicationStartTimeStamp = fetcher.getReplicationStartTimeStamp(); + if (replicationStartTimeStamp != null) { + slave.add("replicationStartTime", replicationStartTimeStamp.toString()); + } + long elapsed = fetcher.getReplicationTimeElapsed(); + slave.add("timeElapsed", String.valueOf(elapsed) + "s"); + + if (bytesDownloaded > 0) + estimatedTimeRemaining = ((bytesToDownload - bytesDownloaded) * elapsed) / bytesDownloaded; + float totalPercent = 0; + long downloadSpeed = 0; + if (bytesToDownload > 0) + totalPercent = (bytesDownloaded * 100) / bytesToDownload; + if (elapsed > 0) + downloadSpeed = (bytesDownloaded / elapsed); + if (currFile != null) + slave.add("currentFile", currFile); + slave.add("currentFileSize", NumberUtils.readableSize(currFileSize)); + slave.add("currentFileSizeDownloaded", NumberUtils.readableSize(currFileSizeDownloaded)); + slave.add("currentFileSizePercent", String.valueOf(percentDownloaded)); + slave.add("bytesDownloaded", NumberUtils.readableSize(bytesDownloaded)); + slave.add("totalPercent", String.valueOf(totalPercent)); + slave.add("timeRemaining", String.valueOf(estimatedTimeRemaining) + "s"); + slave.add("downloadSpeed", NumberUtils.readableSize(downloadSpeed)); + } catch (Exception e) { + LOG.error("Exception while writing replication details: ", e); + } + } + } + + if (isMaster) + details.add("master", master); + if (slave.size() > 0) + details.add("slave", slave); + + NamedList snapshotStats = snapShootDetails; + if (snapshotStats != null) + details.add(CMD_BACKUP, snapshotStats); + + return details; + } + + private void addVal(NamedList nl, String key, Properties props, Class clzz) { + String s = props.getProperty(key); + if (s == null || s.trim().length() == 0) return; + if (clzz == Date.class) { + try { + Long l = Long.parseLong(s); + nl.add(key, new Date(l).toString()); + } catch (NumberFormatException e) {/*no op*/ } + } else if (clzz == List.class) { + String ss[] = s.split(","); + List l = new ArrayList<>(); + for (String s1 : ss) { + l.add(new Date(Long.parseLong(s1)).toString()); + } + nl.add(key, l); + } else { + nl.add(key, s); + } + + } + + private List getReplicateAfterStrings() { + List replicateAfter = new ArrayList<>(); + if (replicateOnCommit) + replicateAfter.add("commit"); + if (replicateOnOptimize) + replicateAfter.add("optimize"); + if (replicateOnStart) + replicateAfter.add("startup"); + return replicateAfter; + } + + Properties loadReplicationProperties() { + Directory dir = null; + try { + try { + dir = core.getDirectoryFactory().get(core.getDataDir(), + DirContext.META_DATA, core.getSolrConfig().indexConfig.lockType); + IndexInput input; + try { + input = dir.openInput( + IndexFetcher.REPLICATION_PROPERTIES, IOContext.DEFAULT); + } catch (FileNotFoundException | NoSuchFileException e) { + return new Properties(); + } + + try { + final InputStream is = new PropertiesInputStream(input); + Properties props = new Properties(); + props.load(new InputStreamReader(is, StandardCharsets.UTF_8)); + return props; + } finally { + input.close(); + } + } finally { + if (dir != null) { + core.getDirectoryFactory().release(dir); + } + } + } catch (IOException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, e); + } + } + + +// void refreshCommitpoint() { +// IndexCommit commitPoint = core.getDeletionPolicy().getLatestCommit(); +// if(replicateOnCommit || (replicateOnOptimize && commitPoint.getSegmentCount() == 1)) { +// indexCommitPoint = commitPoint; +// } +// } + + private void setupPolling(String intervalStr) { + pollIntervalStr = intervalStr; + pollIntervalNs = readIntervalNs(pollIntervalStr); + if (pollIntervalNs == null || pollIntervalNs <= 0) { + LOG.info(" No value set for 'pollInterval'. Timer Task not started."); + return; + } + + Runnable task = () -> { + if (pollDisabled.get()) { + LOG.info("Poll disabled"); + return; + } + try { + LOG.debug("Polling for index modifications"); + markScheduledExecutionStart(); + doFetch(null, false).getSuccessful(); + } catch (Exception e) { + LOG.error("Exception in fetching index", e); + } + }; + executorService = Executors.newSingleThreadScheduledExecutor( + new DefaultSolrThreadFactory("indexFetcher")); + // Randomize initial delay, with a minimum of 1ms + long initialDelayNs = new Random().nextLong() % pollIntervalNs + + TimeUnit.NANOSECONDS.convert(1, TimeUnit.MILLISECONDS); + executorService.scheduleAtFixedRate(task, initialDelayNs, pollIntervalNs, TimeUnit.NANOSECONDS); + LOG.info("Poll scheduled at an interval of {}ms", + TimeUnit.MILLISECONDS.convert(pollIntervalNs, TimeUnit.NANOSECONDS)); + } + + //TODO: CUSTOM + private File contentStorePath; + public static final String CONTENT_STORE_FILE = "csf"; + + @Override + public void init(NamedList args) { + super.init(args); + NamedList configuration = (NamedList)ofNullable(initArgs.get("master")).orElse(initArgs.get("slave")); + } + + @Override + @SuppressWarnings("unchecked") + public void inform(SolrCore core) { + this.core = core; + registerCloseHook(); + Object nbtk = initArgs.get(NUMBER_BACKUPS_TO_KEEP_INIT_PARAM); + if(nbtk!=null) { + numberBackupsToKeep = Integer.parseInt(nbtk.toString()); + } else { + numberBackupsToKeep = 0; + } + NamedList slave = (NamedList) initArgs.get("slave"); + boolean enableSlave = isEnabled( slave ); + if (enableSlave) { + currentIndexFetcher = pollingIndexFetcher = new IndexFetcher(slave, this, core); + setupPolling((String) slave.get(POLL_INTERVAL)); + isSlave = true; + } + NamedList master = (NamedList) initArgs.get("master"); + boolean enableMaster = isEnabled( master ); + + if (enableMaster || enableSlave) { + if (core.getCoreContainer().getZkController() != null) { + LOG.warn("SolrCloud is enabled for core " + core.getName() + " but so is old-style replication. Make sure you" + + " intend this behavior, it usually indicates a mis-configuration. Master setting is " + + Boolean.toString(enableMaster) + " and slave setting is " + Boolean.toString(enableSlave)); + } + } + + if (!enableSlave && !enableMaster) { + enableMaster = true; + master = new NamedList<>(); + } + + if (enableMaster) { + includeConfFiles = (String) master.get(CONF_FILES); + if (includeConfFiles != null && includeConfFiles.trim().length() > 0) { + List files = Arrays.asList(includeConfFiles.split(",")); + for (String file : files) { + if (file.trim().length() == 0) continue; + String[] strs = file.trim().split(":"); + // if there is an alias add it or it is null + confFileNameAlias.add(strs[0], strs.length > 1 ? strs[1] : null); + } + LOG.info("Replication enabled for following config files: " + includeConfFiles); + } + List backup = master.getAll("backupAfter"); + boolean backupOnCommit = backup.contains("commit"); + boolean backupOnOptimize = !backupOnCommit && backup.contains("optimize"); + List replicateAfter = master.getAll(REPLICATE_AFTER); + replicateOnCommit = replicateAfter.contains("commit"); + replicateOnOptimize = !replicateOnCommit && replicateAfter.contains("optimize"); + + if (!replicateOnCommit && ! replicateOnOptimize) { + replicateOnCommit = true; + } + + // if we only want to replicate on optimize, we need the deletion policy to + // save the last optimized commit point. + if (replicateOnOptimize) { + IndexDeletionPolicyWrapper wrapper = core.getDeletionPolicy(); + IndexDeletionPolicy policy = wrapper == null ? null : wrapper.getWrappedDeletionPolicy(); + if (policy instanceof SolrDeletionPolicy) { + SolrDeletionPolicy solrPolicy = (SolrDeletionPolicy)policy; + if (solrPolicy.getMaxOptimizedCommitsToKeep() < 1) { + solrPolicy.setMaxOptimizedCommitsToKeep(1); + } + } else { + LOG.warn("Replication can't call setMaxOptimizedCommitsToKeep on " + policy); + } + } + + if (replicateOnOptimize || backupOnOptimize) { + core.getUpdateHandler().registerOptimizeCallback(getEventListener(backupOnOptimize, replicateOnOptimize)); + } + if (replicateOnCommit || backupOnCommit) { + replicateOnCommit = true; + core.getUpdateHandler().registerCommitCallback(getEventListener(backupOnCommit, replicateOnCommit)); + } + if (replicateAfter.contains("startup")) { + replicateOnStart = true; + RefCounted s = core.getNewestSearcher(false); + try { + DirectoryReader reader = s==null ? null : s.get().getIndexReader(); + if (reader!=null && reader.getIndexCommit() != null && reader.getIndexCommit().getGeneration() != 1L) { + try { + if(replicateOnOptimize){ + Collection commits = DirectoryReader.listCommits(reader.directory()); + for (IndexCommit ic : commits) { + if(ic.getSegmentCount() == 1){ + if(indexCommitPoint == null || indexCommitPoint.getGeneration() < ic.getGeneration()) indexCommitPoint = ic; + } + } + } else{ + indexCommitPoint = reader.getIndexCommit(); + } + } finally { + // We don't need to save commit points for replication, the SolrDeletionPolicy + // always saves the last commit point (and the last optimized commit point, if needed) + /*** + if(indexCommitPoint != null){ + core.getDeletionPolicy().saveCommitPoint(indexCommitPoint.getGeneration()); + } + ***/ + } + } + + // ensure the writer is init'd so that we have a list of commit points + RefCounted iw = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core); + iw.decref(); + + } catch (IOException e) { + LOG.warn("Unable to get IndexCommit on startup", e); + } finally { + if (s!=null) s.decref(); + } + } + String reserve = (String) master.get(RESERVE); + if (reserve != null && !reserve.trim().equals("")) { + reserveCommitDuration = readIntervalMs(reserve); + } + LOG.info("Commits will be reserved for " + reserveCommitDuration); + isMaster = true; + } + } + + // check master or slave is enabled + private boolean isEnabled( NamedList params ){ + if( params == null ) return false; + Object enable = params.get( "enable" ); + if( enable == null ) return true; + if( enable instanceof String ) + return StrUtils.parseBool( (String)enable ); + return Boolean.TRUE.equals( enable ); + } + + /** + * register a closehook + */ + private void registerCloseHook() { + core.addCloseHook(new CloseHook() { + @Override + public void preClose(SolrCore core) { + try { + if (executorService != null) executorService.shutdown(); // we don't wait for shutdown - this can deadlock core reload + } finally { + if (pollingIndexFetcher != null) { + pollingIndexFetcher.destroy(); + } + } + if (currentIndexFetcher != null && currentIndexFetcher != pollingIndexFetcher) { + currentIndexFetcher.destroy(); + } + } + + @Override + public void postClose(SolrCore core) {} + }); + + core.addCloseHook(new CloseHook() { + @Override + public void preClose(SolrCore core) { + ExecutorUtil.shutdownAndAwaitTermination(restoreExecutor); + if (restoreFuture != null) { + restoreFuture.cancel(false); + } + } + + @Override + public void postClose(SolrCore core) {} + }); + } + + /** + * Register a listener for postcommit/optimize + * + * @param snapshoot do a snapshoot + * @param getCommit get a commitpoint also + * + * @return an instance of the eventlistener + */ + private SolrEventListener getEventListener(final boolean snapshoot, final boolean getCommit) { + return new SolrEventListener() { + @Override + public void init(NamedList args) {/*no op*/ } + + /** + * This refreshes the latest replicateable index commit and optionally can create Snapshots as well + */ + @Override + public void postCommit() { + IndexCommit currentCommitPoint = core.getDeletionPolicy().getLatestCommit(); + + if (getCommit) { + // IndexCommit oldCommitPoint = indexCommitPoint; + indexCommitPoint = currentCommitPoint; + + // We don't need to save commit points for replication, the SolrDeletionPolicy + // always saves the last commit point (and the last optimized commit point, if needed) + /*** + if (indexCommitPoint != null) { + core.getDeletionPolicy().saveCommitPoint(indexCommitPoint.getGeneration()); + } + if(oldCommitPoint != null){ + core.getDeletionPolicy().releaseCommitPointAndExtendReserve(oldCommitPoint.getGeneration()); + } + ***/ + } + if (snapshoot) { + try { + int numberToKeep = numberBackupsToKeep; + if (numberToKeep < 1) { + numberToKeep = Integer.MAX_VALUE; + } + SnapShooter snapShooter = new SnapShooter(core, null, null); + snapShooter.validateCreateSnapshot(); + snapShooter.createSnapAsync(currentCommitPoint, numberToKeep, (nl) -> snapShootDetails = nl); + } catch (Exception e) { + LOG.error("Exception while snapshooting", e); + } + } + } + + @Override + public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) { /*no op*/} + + @Override + public void postSoftCommit() { + + } + }; + } + + /**This class is used to read and send files in the lucene index + * + */ + private class DirectoryFileStream implements SolrCore.RawWriter { + protected SolrParams params; + + protected FastOutputStream fos; + + protected Long indexGen; + protected IndexDeletionPolicyWrapper delPolicy; + + protected String fileName; + protected String cfileName; + protected String tlogFileName; + protected String contentStoreFilename; + protected String sOffset; + protected String sLen; + protected String compress; + protected boolean useChecksum; + + protected long offset = -1; + protected int len = -1; + + protected Checksum checksum; + + private RateLimiter rateLimiter; + + byte[] buf; + + public DirectoryFileStream(SolrParams solrParams) { + params = solrParams; + delPolicy = core.getDeletionPolicy(); + + fileName = validateFilenameOrError(params.get(FILE)); + cfileName = validateFilenameOrError(params.get(CONF_FILE_SHORT)); + tlogFileName = validateFilenameOrError(params.get(TLOG_FILE)); + contentStoreFilename = validateFilenameOrError(params.get(CONTENT_STORE_FILE)); + + sOffset = params.get(OFFSET); + sLen = params.get(LEN); + compress = params.get(COMPRESSION); + useChecksum = params.getBool(CHECKSUM, false); + + indexGen = params.getLong(GENERATION); + if (useChecksum) { + checksum = new Adler32(); + } + //No throttle if MAX_WRITE_PER_SECOND is not specified + double maxWriteMBPerSec = params.getDouble(MAX_WRITE_PER_SECOND, Double.MAX_VALUE); + rateLimiter = new RateLimiter.SimpleRateLimiter(maxWriteMBPerSec); + } + + // Throw exception on directory traversal attempts + protected String validateFilenameOrError(String filename) { + if (filename != null) { + Path filePath = Paths.get(filename); + filePath.forEach(subpath -> { + if ("..".equals(subpath.toString())) { + throw new SolrException(ErrorCode.FORBIDDEN, "File name cannot contain .."); + } + }); + if (filePath.isAbsolute()) { + throw new SolrException(ErrorCode.FORBIDDEN, "File name must be relative"); + } + return filename; + } else return null; + } + + protected void initWrite() throws IOException { + if (sOffset != null) offset = Long.parseLong(sOffset); + if (sLen != null) len = Integer.parseInt(sLen); + if (fileName == null && cfileName == null && tlogFileName == null && contentStoreFilename == null) { + // no filename do nothing + writeNothingAndFlush(); + } + buf = new byte[(len == -1 || len > PACKET_SZ) ? PACKET_SZ : len]; + + //reserve commit point till write is complete + if(indexGen != null) { + delPolicy.saveCommitPoint(indexGen); + } + } + + protected void createOutputStream(OutputStream out) { + if (Boolean.parseBoolean(compress)) { + fos = new FastOutputStream(new DeflaterOutputStream(out)); + } else { + fos = new FastOutputStream(out); + } + } + + protected void extendReserveAndReleaseCommitPoint() { + if(indexGen != null) { + //Reserve the commit point for another 10s for the next file to be to fetched. + //We need to keep extending the commit reservation between requests so that the replica can fetch + //all the files correctly. + delPolicy.setReserveDuration(indexGen, reserveCommitDuration); + + //release the commit point as the write is complete + delPolicy.releaseCommitPoint(indexGen); + } + + } + public void write(OutputStream out) throws IOException { + createOutputStream(out); + + IndexInput in = null; + try { + initWrite(); + + RefCounted sref = core.getSearcher(); + Directory dir; + try { + SolrIndexSearcher searcher = sref.get(); + dir = searcher.getIndexReader().directory(); + } finally { + sref.decref(); + } + in = dir.openInput(fileName, IOContext.READONCE); + // if offset is mentioned move the pointer to that point + if (offset != -1) in.seek(offset); + + long filelen = dir.fileLength(fileName); + long maxBytesBeforePause = 0; + + while (true) { + offset = offset == -1 ? 0 : offset; + int read = (int) Math.min(buf.length, filelen - offset); + in.readBytes(buf, 0, read); + + fos.writeInt(read); + if (useChecksum) { + checksum.reset(); + checksum.update(buf, 0, read); + fos.writeLong(checksum.getValue()); + } + fos.write(buf, 0, read); + fos.flush(); + LOG.debug("Wrote {} bytes for file {}", offset + read, fileName); + + //Pause if necessary + maxBytesBeforePause += read; + if (maxBytesBeforePause >= rateLimiter.getMinPauseCheckBytes()) { + rateLimiter.pause(maxBytesBeforePause); + maxBytesBeforePause = 0; + } + if (read != buf.length) { + writeNothingAndFlush(); + fos.close(); + break; + } + offset += read; + in.seek(offset); + } + } catch (IOException e) { + LOG.warn("Exception while writing response for params: " + params, e); + } finally { + if (in != null) { + in.close(); + } + extendReserveAndReleaseCommitPoint(); + } + } + + + /** + * Used to write a marker for EOF + */ + protected void writeNothingAndFlush() throws IOException { + fos.writeInt(0); + fos.flush(); + } + } + + /**This is used to write files in the conf directory. + */ + private abstract class LocalFsFileStream extends DirectoryFileStream { + + private File file; + + public LocalFsFileStream(SolrParams solrParams) { + super(solrParams); + this.file = this.initFile(); + } + + protected abstract File initFile(); + + @Override + public void write(OutputStream out) throws IOException { + createOutputStream(out); + FileInputStream inputStream = null; + try { + initWrite(); + + if (file.exists() && file.canRead()) { + inputStream = new FileInputStream(file); + FileChannel channel = inputStream.getChannel(); + //if offset is mentioned move the pointer to that point + if (offset != -1) + channel.position(offset); + ByteBuffer bb = ByteBuffer.wrap(buf); + + while (true) { + bb.clear(); + long bytesRead = channel.read(bb); + if (bytesRead <= 0) { + writeNothingAndFlush(); + fos.close(); + break; + } + fos.writeInt((int) bytesRead); + if (useChecksum) { + checksum.reset(); + checksum.update(buf, 0, (int) bytesRead); + fos.writeLong(checksum.getValue()); + } + fos.write(buf, 0, (int) bytesRead); + fos.flush(); + } + } else { + writeNothingAndFlush(); + } + } catch (IOException e) { + LOG.warn("Exception while writing response for params: " + params, e); + } finally { + IOUtils.closeQuietly(inputStream); + extendReserveAndReleaseCommitPoint(); + } + } + } + + private class ContentStoreFilesStream extends LocalFsFileStream { + public ContentStoreFilesStream(SolrParams solrParams) { + super(solrParams); + } + + protected File initFile() { + return new File("some");//FIXME + } + + + @Override + public void write(OutputStream out) throws IOException { + createOutputStream(out); + FileInputStream inputStream = null; + String contentStoreRoot = ContentStoreCache.get().getContentStoreRootPath(); + try { + for (String fileName : params.getParams(CONTENT_STORE_FILE_LIST)) + { + Map fileMeta = new HashMap<>(); + File f = new File(contentStoreRoot + fileName); + if (f.exists() && !f.isDirectory()) { + try { + writeFile(f, fileName); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + fos.close(); + extendReserveAndReleaseCommitPoint(); + } + } + + + @Override + protected void writeNothingAndFlush() throws IOException { + fos.flush(); + } + + public void writeFile(File file, String fileName) throws IOException { + + buf = new byte[PACKET_SZ]; + if (file.exists() && file.canRead()) { + FileInputStream inputStream = new FileInputStream(file); + FileChannel channel = inputStream.getChannel(); + //if offset is mentioned move the pointer to that point + if (offset != -1) + channel.position(offset); + ByteBuffer bb = ByteBuffer.wrap(buf); + channel.size(); + + fos.writeInt( fileName.length()); + fos.write(fileName.getBytes()); + fos.writeInt((int) channel.size()); + + if (channel.size() != 0) { + + while (true) { + bb.clear(); + long bytesRead = channel.read(bb); + if (bytesRead <= 0) { + writeNothingAndFlush(); + break; + } + + fos.write(buf, 0, (int) bytesRead); + fos.flush(); + } + } + } + } + } + + + private class LocalContentStoreFileStream extends LocalFsFileStream { + public LocalContentStoreFileStream(SolrParams solrParams) { + super(solrParams); + } + + protected File initFile() { + return new File(ContentStoreCache.get().getContentStoreRootPath() + "/" + contentStoreFilename); + } + + } + + + private class LocalFsTlogFileStream extends LocalFsFileStream { + + public LocalFsTlogFileStream(SolrParams solrParams) { + super(solrParams); + } + + protected File initFile() { + //if it is a tlog file read from tlog directory + return new File(core.getUpdateHandler().getUpdateLog().getLogDir(), tlogFileName); + } + + } + + private class LocalFsConfFileStream extends LocalFsFileStream { + + public LocalFsConfFileStream(SolrParams solrParams) { + super(solrParams); + } + + protected File initFile() { + //if it is a conf file read from config directory + return new File(core.getResourceLoader().getConfigDir(), cfileName); + } + + } + + private static Integer readIntervalMs(String interval) { + return (int) TimeUnit.MILLISECONDS.convert(readIntervalNs(interval), TimeUnit.NANOSECONDS); + } + + private static Long readIntervalNs(String interval) { + if (interval == null) + return null; + int result = 0; + Matcher m = INTERVAL_PATTERN.matcher(interval.trim()); + if (m.find()) { + String hr = m.group(1); + String min = m.group(2); + String sec = m.group(3); + result = 0; + try { + if (sec != null && sec.length() > 0) + result += Integer.parseInt(sec); + if (min != null && min.length() > 0) + result += (60 * Integer.parseInt(min)); + if (hr != null && hr.length() > 0) + result += (60 * 60 * Integer.parseInt(hr)); + return TimeUnit.NANOSECONDS.convert(result, TimeUnit.SECONDS); + } catch (NumberFormatException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, INTERVAL_ERR_MSG); + } + } else { + throw new SolrException(ErrorCode.SERVER_ERROR, INTERVAL_ERR_MSG); + } + } + + private static final String SUCCESS = "success"; + + private static final String FAILED = "failed"; + + private static final String EXCEPTION = "exception"; + + public static final String MASTER_URL = "masterUrl"; + + public static final String STATUS = "status"; + + public static final String COMMAND = "command"; + + public static final String CMD_DETAILS = "details"; + + public static final String CMD_BACKUP = "backup"; + + public static final String CMD_RESTORE = "restore"; + + public static final String CMD_RESTORE_STATUS = "restorestatus"; + + public static final String CMD_FETCH_INDEX = "fetchindex"; + + public static final String CMD_ABORT_FETCH = "abortfetch"; + + public static final String CMD_GET_FILE_LIST = "filelist"; + + public static final String CMD_GET_FILE = "filecontent"; + + public static final String CMD_DISABLE_POLL = "disablepoll"; + + public static final String CMD_DISABLE_REPL = "disablereplication"; + + public static final String CMD_ENABLE_REPL = "enablereplication"; + + public static final String CMD_ENABLE_POLL = "enablepoll"; + + public static final String CMD_INDEX_VERSION = "indexversion"; + + public static final String CMD_SHOW_COMMITS = "commits"; + + public static final String CMD_DELETE_BACKUP = "deletebackup"; + + public static final String GENERATION = "generation"; + + public static final String OFFSET = "offset"; + + public static final String LEN = "len"; + + public static final String FILE = "file"; + + public static final String SIZE = "size"; + + public static final String MAX_WRITE_PER_SECOND = "maxWriteMBPerSec"; + + public static final String CONF_FILE_SHORT = "cf"; + + public static final String TLOG_FILE = "tlogFile"; + + public static final String CHECKSUM = "checksum"; + + public static final String ALIAS = "alias"; + + public static final String CONF_CHECKSUM = "confchecksum"; + + public static final String CONF_FILES = "confFiles"; + + public static final String TLOG_FILES = "tlogFiles"; + + public static final String CONTENT_STORE_FILES = "contentStoreFiles"; + + public static final String REPLICATE_AFTER = "replicateAfter"; + + public static final String FILE_STREAM = "filestream"; + + public static final String POLL_INTERVAL = "pollInterval"; + + public static final String INTERVAL_ERR_MSG = "The " + POLL_INTERVAL + " must be in this format 'HH:mm:ss'"; + + private static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)"); + + public static final int PACKET_SZ = 1024 * 1024; // 1MB + + public static final String RESERVE = "commitReserveDuration"; + + public static final String COMPRESSION = "compression"; + + public static final String EXTERNAL = "external"; + + public static final String INTERNAL = "internal"; + + public static final String ERR_STATUS = "ERROR"; + + public static final String OK_STATUS = "OK"; + + public static final String NEXT_EXECUTION_AT = "nextExecutionAt"; + + public static final String NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM = "numberToKeep"; + + public static final String NUMBER_BACKUPS_TO_KEEP_INIT_PARAM = "maxNumberOfBackups"; + + public static final String CONTENT_STORE_FILE_LIST = "contentStoreFiles"; + + public static final String CMD_CONTENT_STORE_FILES = "cmdContentStoreFiles"; + + /** + * Boolean param for tests that can be specified when using + * {@link #CMD_FETCH_INDEX} to force the current request to block until + * the fetch is complete. NOTE: This param is not advised for + * non-test code, since the the duration of the fetch for non-trivial + * indexes will likeley cause the request to time out. + * + * @lucene.internal + */ + public static final String WAIT = "wait"; +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java new file mode 100644 index 000000000..d75b94bd1 --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.alfresco.solr.handler; + +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.store.Directory; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.DirectoryFactory.DirContext; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.core.backup.repository.BackupRepository.PathType; +import org.apache.solr.core.backup.repository.LocalFileSystemRepository; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.RefCounted; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; + +/** + *

Provides functionality equivalent to the snapshooter script

+ * This is no longer used in standard replication. + * + * + * @since solr 1.4 + */ +public class SnapShooter { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private SolrCore solrCore; + private String snapshotName = null; + private String directoryName = null; + private URI baseSnapDirPath = null; + private URI snapshotDirPath = null; + private BackupRepository backupRepo = null; + private String commitName; // can be null + + @Deprecated + public SnapShooter(SolrCore core, String location, String snapshotName) { + String snapDirStr = null; + // Note - This logic is only applicable to the usecase where a shared file-system is exposed via + // local file-system interface (primarily for backwards compatibility). For other use-cases, users + // will be required to specify "location" where the backup should be stored. + if (location == null) { + snapDirStr = core.getDataDir(); + } else { + snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString(); + } + initialize(new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null); + } + + public SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) { + initialize(backupRepo, core, location, snapshotName, commitName); + } + + private void initialize(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) { + this.solrCore = Objects.requireNonNull(core); + this.backupRepo = Objects.requireNonNull(backupRepo); + this.baseSnapDirPath = location; + this.snapshotName = snapshotName; + if (snapshotName != null) { + directoryName = "snapshot." + snapshotName; + } else { + SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.ROOT); + directoryName = "snapshot." + fmt.format(new Date()); + } + this.snapshotDirPath = backupRepo.resolve(location, directoryName); + this.commitName = commitName; + } + + public BackupRepository getBackupRepository() { + return backupRepo; + } + + /** + * Gets the parent directory of the snapshots. This is the {@code location} + * given in the constructor. + */ + public URI getLocation() { + return this.baseSnapDirPath; + } + + public void validateDeleteSnapshot() { + Objects.requireNonNull(this.snapshotName); + + boolean dirFound = false; + String[] paths; + try { + paths = backupRepo.listAll(baseSnapDirPath); + for (String path : paths) { + if (path.equals(this.directoryName) + && backupRepo.getPathType(baseSnapDirPath.resolve(path)) == PathType.DIRECTORY) { + dirFound = true; + break; + } + } + if(dirFound == false) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Snapshot " + snapshotName + " cannot be found in directory: " + baseSnapDirPath); + } + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to find snapshot " + snapshotName + " in directory: " + baseSnapDirPath, e); + } + } + + protected void deleteSnapAsync(final ReplicationHandler replicationHandler) { + new Thread(() -> deleteNamedSnapshot(replicationHandler)).start(); + } + + public void validateCreateSnapshot() throws IOException { + // Note - Removed the current behavior of creating the directory hierarchy. + // Do we really need to provide this support? + if (!backupRepo.exists(baseSnapDirPath)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + " Directory does not exist: " + snapshotDirPath); + } + + if (backupRepo.exists(snapshotDirPath)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Snapshot directory already exists: " + snapshotDirPath); + } + } + + public NamedList createSnapshot() throws Exception { + RefCounted searcher = solrCore.getSearcher(); + try { + if (commitName != null) { + SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager(); + Optional commit = snapshotMgr.getIndexCommitByName(commitName); + if(commit.isPresent()) { + return createSnapshot(commit.get()); + } + throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to find an index commit with name " + commitName + + " for core " + solrCore.getName()); + } else { + //TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first? + IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy(); + IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit(); + deletionPolicy.saveCommitPoint(indexCommit.getGeneration()); + try { + return createSnapshot(indexCommit); + } finally { + deletionPolicy.releaseCommitPoint(indexCommit.getGeneration()); + } + } + } finally { + searcher.decref(); + } + } + + public void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, Consumer result) { + solrCore.getDeletionPolicy().saveCommitPoint(indexCommit.getGeneration()); + + //TODO should use Solr's ExecutorUtil + new Thread(() -> { + try { + result.accept(createSnapshot(indexCommit)); + } catch (Exception e) { + LOG.error("Exception while creating snapshot", e); + NamedList snapShootDetails = new NamedList<>(); + snapShootDetails.add("snapShootException", e.getMessage()); + result.accept(snapShootDetails); + } finally { + solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration()); + } + if (snapshotName == null) { + try { + deleteOldBackups(numberToKeep); + } catch (IOException e) { + LOG.warn("Unable to delete old snapshots ", e); + } + } + }).start(); + + } + + // note: remember to reserve the indexCommit first so it won't get deleted concurrently + protected NamedList createSnapshot(final IndexCommit indexCommit) throws Exception { + LOG.info("Creating backup snapshot " + (snapshotName == null ? "" : snapshotName) + " at " + baseSnapDirPath); + boolean success = false; + try { + NamedList details = new NamedList<>(); + details.add("startTime", new Date().toString());//bad; should be Instant.now().toString() + + Collection files = indexCommit.getFileNames(); + Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); + try { + for(String fileName : files) { + backupRepo.copyFileFrom(dir, fileName, snapshotDirPath); + } + } finally { + solrCore.getDirectoryFactory().release(dir); + } + + details.add("fileCount", files.size()); + details.add("status", "success"); + details.add("snapshotCompletedAt", new Date().toString());//bad; should be Instant.now().toString() + details.add("snapshotName", snapshotName); + LOG.info("Done creating backup snapshot: " + (snapshotName == null ? "" : snapshotName) + + " at " + baseSnapDirPath); + success = true; + return details; + } finally { + if (!success) { + try { + backupRepo.deleteDirectory(snapshotDirPath); + } catch (Exception excDuringDelete) { + LOG.warn("Failed to delete "+snapshotDirPath+" after snapshot creation failed due to: "+excDuringDelete); + } + } + } + } + + private void deleteOldBackups(int numberToKeep) throws IOException { + String[] paths = backupRepo.listAll(baseSnapDirPath); + List dirs = new ArrayList<>(); + for (String f : paths) { + if (backupRepo.getPathType(baseSnapDirPath.resolve(f)) == PathType.DIRECTORY) { + OldBackupDirectory obd = new OldBackupDirectory(baseSnapDirPath, f); + if (obd.getTimestamp().isPresent()) { + dirs.add(obd); + } + } + } + if (numberToKeep > dirs.size() -1) { + return; + } + Collections.sort(dirs); + int i=1; + for (OldBackupDirectory dir : dirs) { + if (i++ > numberToKeep) { + backupRepo.deleteDirectory(dir.getPath()); + } + } + } + + protected void deleteNamedSnapshot(ReplicationHandler replicationHandler) { + LOG.info("Deleting snapshot: " + snapshotName); + + NamedList details = new NamedList<>(); + + try { + URI path = baseSnapDirPath.resolve("snapshot." + snapshotName); + backupRepo.deleteDirectory(path); + + details.add("status", "success"); + details.add("snapshotDeletedAt", new Date().toString()); + + } catch (IOException e) { + details.add("status", "Unable to delete snapshot: " + snapshotName); + LOG.warn("Unable to delete snapshot: " + snapshotName, e); + } + + replicationHandler.snapShootDetails = details; + } + + public static final String DATE_FMT = "yyyyMMddHHmmssSSS"; + +} From 436df479664392eea5918addcd1de321039e5981 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Wed, 25 Sep 2019 11:32:13 +0200 Subject: [PATCH 02/76] [SEARCH-1862] draft ContentStoreCache --- .../solr/content/ContentStoreCache.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ContentStoreCache.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ContentStoreCache.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ContentStoreCache.java new file mode 100644 index 000000000..a19fc0c81 --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ContentStoreCache.java @@ -0,0 +1,40 @@ +package org.alfresco.solr.content; + +import java.util.List; + +public class ContentStoreCache { + + private static ContentStoreCache contentStoreCache = null; + + private List transactions; + + private String root; + + public void init(String root){ + this.root = root; + } + + public String getContentStoreRootPath() { + return this.root; + } + + public synchronized static ContentStoreCache get(){ + if (contentStoreCache == null) + { + contentStoreCache = new ContentStoreCache(); + } + + return contentStoreCache; + } + + public class ContentStoreTransaction { + private Long txId; + private List deletion; + private List updates; + } + + public class ContentStoreCacheEntry { + public long dbid; + public long hash; + } +} From 92c1a00903d8d7518de182ffb537684af8971281 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Wed, 25 Sep 2019 11:32:41 +0200 Subject: [PATCH 03/76] [SEARCH-1862] Activated custom replication handler in solrconfig --- .../solr/instance/templates/rerank/conf/solrconfig.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml index 42eb819bb..f41aac283 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml @@ -1154,7 +1154,8 @@ https://wiki.apache.org/solr/SolrCloud/ --> - + + + + + + + + + + + + + + + + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ca.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ca.txt new file mode 100644 index 000000000..307a85f91 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ca.txt @@ -0,0 +1,8 @@ +# Set of Catalan contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +d +l +m +n +s +t diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_fr.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_fr.txt new file mode 100644 index 000000000..f1bba51b2 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_fr.txt @@ -0,0 +1,15 @@ +# Set of French contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +l +m +t +qu +n +s +j +d +c +jusqu +quoiqu +lorsqu +puisqu diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ga.txt new file mode 100644 index 000000000..9ebe7fa34 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ga.txt @@ -0,0 +1,5 @@ +# Set of Irish contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +d +m +b diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_it.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_it.txt new file mode 100644 index 000000000..cac040953 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_it.txt @@ -0,0 +1,23 @@ +# Set of Italian contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +c +l +all +dall +dell +nell +sull +coll +pell +gl +agl +dagl +degl +negl +sugl +un +m +t +s +v +d diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/hyphenations_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/hyphenations_ga.txt new file mode 100644 index 000000000..4d2642cc5 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/hyphenations_ga.txt @@ -0,0 +1,5 @@ +# Set of Irish hyphenations for StopFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +h +n +t diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stemdict_nl.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stemdict_nl.txt new file mode 100644 index 000000000..441072971 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stemdict_nl.txt @@ -0,0 +1,6 @@ +# Set of overrides for the dutch stemmer +# TODO: load this as a resource from the analyzer and sync it in build.xml +fiets fiets +bromfiets bromfiets +ei eier +kind kinder diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stoptags_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stoptags_ja.txt new file mode 100644 index 000000000..71b750845 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stoptags_ja.txt @@ -0,0 +1,420 @@ +# +# This file defines a Japanese stoptag set for JapanesePartOfSpeechStopFilter. +# +# Any token with a part-of-speech tag that exactly matches those defined in this +# file are removed from the token stream. +# +# Set your own stoptags by uncommenting the lines below. Note that comments are +# not allowed on the same line as a stoptag. See LUCENE-3745 for frequency lists, +# etc. that can be useful for building you own stoptag set. +# +# The entire possible tagset is provided below for convenience. +# +##### +# noun: unclassified nouns +#名詞 +# +# noun-common: Common nouns or nouns where the sub-classification is undefined +#名詞-一般 +# +# noun-proper: Proper nouns where the sub-classification is undefined +#名詞-固有名詞 +# +# noun-proper-misc: miscellaneous proper nouns +#名詞-固有名詞-一般 +# +# noun-proper-person: Personal names where the sub-classification is undefined +#名詞-固有名詞-人名 +# +# noun-proper-person-misc: names that cannot be divided into surname and +# given name; foreign names; names where the surname or given name is unknown. +# e.g. お市の方 +#名詞-固有名詞-人名-一般 +# +# noun-proper-person-surname: Mainly Japanese surnames. +# e.g. 山田 +#名詞-固有名詞-人名-姓 +# +# noun-proper-person-given_name: Mainly Japanese given names. +# e.g. 太郎 +#名詞-固有名詞-人名-名 +# +# noun-proper-organization: Names representing organizations. +# e.g. 通産省, NHK +#名詞-固有名詞-組織 +# +# noun-proper-place: Place names where the sub-classification is undefined +#名詞-固有名詞-地域 +# +# noun-proper-place-misc: Place names excluding countries. +# e.g. アジア, バルセロナ, 京都 +#名詞-固有名詞-地域-一般 +# +# noun-proper-place-country: Country names. +# e.g. 日本, オーストラリア +#名詞-固有名詞-地域-国 +# +# noun-pronoun: Pronouns where the sub-classification is undefined +#名詞-代名詞 +# +# noun-pronoun-misc: miscellaneous pronouns: +# e.g. それ, ここ, あいつ, あなた, あちこち, いくつ, どこか, なに, みなさん, みんな, わたくし, われわれ +#名詞-代名詞-一般 +# +# noun-pronoun-contraction: Spoken language contraction made by combining a +# pronoun and the particle 'wa'. +# e.g. ありゃ, こりゃ, こりゃあ, そりゃ, そりゃあ +#名詞-代名詞-縮約 +# +# noun-adverbial: Temporal nouns such as names of days or months that behave +# like adverbs. Nouns that represent amount or ratios and can be used adverbially, +# e.g. 金曜, 一月, 午後, 少量 +#名詞-副詞可能 +# +# noun-verbal: Nouns that take arguments with case and can appear followed by +# 'suru' and related verbs (する, できる, なさる, くださる) +# e.g. インプット, 愛着, 悪化, 悪戦苦闘, 一安心, 下取り +#名詞-サ変接続 +# +# noun-adjective-base: The base form of adjectives, words that appear before な ("na") +# e.g. 健康, 安易, 駄目, だめ +#名詞-形容動詞語幹 +# +# noun-numeric: Arabic numbers, Chinese numerals, and counters like 何 (回), 数. +# e.g. 0, 1, 2, 何, 数, 幾 +#名詞-数 +# +# noun-affix: noun affixes where the sub-classification is undefined +#名詞-非自立 +# +# noun-affix-misc: Of adnominalizers, the case-marker の ("no"), and words that +# attach to the base form of inflectional words, words that cannot be classified +# into any of the other categories below. This category includes indefinite nouns. +# e.g. あかつき, 暁, かい, 甲斐, 気, きらい, 嫌い, くせ, 癖, こと, 事, ごと, 毎, しだい, 次第, +# 順, せい, 所為, ついで, 序で, つもり, 積もり, 点, どころ, の, はず, 筈, はずみ, 弾み, +# 拍子, ふう, ふり, 振り, ほう, 方, 旨, もの, 物, 者, ゆえ, 故, ゆえん, 所以, わけ, 訳, +# わり, 割り, 割, ん-口語/, もん-口語/ +#名詞-非自立-一般 +# +# noun-affix-adverbial: noun affixes that that can behave as adverbs. +# e.g. あいだ, 間, あげく, 挙げ句, あと, 後, 余り, 以外, 以降, 以後, 以上, 以前, 一方, うえ, +# 上, うち, 内, おり, 折り, かぎり, 限り, きり, っきり, 結果, ころ, 頃, さい, 際, 最中, さなか, +# 最中, じたい, 自体, たび, 度, ため, 為, つど, 都度, とおり, 通り, とき, 時, ところ, 所, +# とたん, 途端, なか, 中, のち, 後, ばあい, 場合, 日, ぶん, 分, ほか, 他, まえ, 前, まま, +# 儘, 侭, みぎり, 矢先 +#名詞-非自立-副詞可能 +# +# noun-affix-aux: noun affixes treated as 助動詞 ("auxiliary verb") in school grammars +# with the stem よう(だ) ("you(da)"). +# e.g. よう, やう, 様 (よう) +#名詞-非自立-助動詞語幹 +# +# noun-affix-adjective-base: noun affixes that can connect to the indeclinable +# connection form な (aux "da"). +# e.g. みたい, ふう +#名詞-非自立-形容動詞語幹 +# +# noun-special: special nouns where the sub-classification is undefined. +#名詞-特殊 +# +# noun-special-aux: The そうだ ("souda") stem form that is used for reporting news, is +# treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the base +# form of inflectional words. +# e.g. そう +#名詞-特殊-助動詞語幹 +# +# noun-suffix: noun suffixes where the sub-classification is undefined. +#名詞-接尾 +# +# noun-suffix-misc: Of the nouns or stem forms of other parts of speech that connect +# to ガル or タイ and can combine into compound nouns, words that cannot be classified into +# any of the other categories below. In general, this category is more inclusive than +# 接尾語 ("suffix") and is usually the last element in a compound noun. +# e.g. おき, かた, 方, 甲斐 (がい), がかり, ぎみ, 気味, ぐるみ, (～した) さ, 次第, 済 (ず) み, +# よう, (でき)っこ, 感, 観, 性, 学, 類, 面, 用 +#名詞-接尾-一般 +# +# noun-suffix-person: Suffixes that form nouns and attach to person names more often +# than other nouns. +# e.g. 君, 様, 著 +#名詞-接尾-人名 +# +# noun-suffix-place: Suffixes that form nouns and attach to place names more often +# than other nouns. +# e.g. 町, 市, 県 +#名詞-接尾-地域 +# +# noun-suffix-verbal: Of the suffixes that attach to nouns and form nouns, those that +# can appear before スル ("suru"). +# e.g. 化, 視, 分け, 入り, 落ち, 買い +#名詞-接尾-サ変接続 +# +# noun-suffix-aux: The stem form of そうだ (様態) that is used to indicate conditions, +# is treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the +# conjunctive form of inflectional words. +# e.g. そう +#名詞-接尾-助動詞語幹 +# +# noun-suffix-adjective-base: Suffixes that attach to other nouns or the conjunctive +# form of inflectional words and appear before the copula だ ("da"). +# e.g. 的, げ, がち +#名詞-接尾-形容動詞語幹 +# +# noun-suffix-adverbial: Suffixes that attach to other nouns and can behave as adverbs. +# e.g. 後 (ご), 以後, 以降, 以前, 前後, 中, 末, 上, 時 (じ) +#名詞-接尾-副詞可能 +# +# noun-suffix-classifier: Suffixes that attach to numbers and form nouns. This category +# is more inclusive than 助数詞 ("classifier") and includes common nouns that attach +# to numbers. +# e.g. 個, つ, 本, 冊, パーセント, cm, kg, カ月, か国, 区画, 時間, 時半 +#名詞-接尾-助数詞 +# +# noun-suffix-special: Special suffixes that mainly attach to inflecting words. +# e.g. (楽し) さ, (考え) 方 +#名詞-接尾-特殊 +# +# noun-suffix-conjunctive: Nouns that behave like conjunctions and join two words +# together. +# e.g. (日本) 対 (アメリカ), 対 (アメリカ), (3) 対 (5), (女優) 兼 (主婦) +#名詞-接続詞的 +# +# noun-verbal_aux: Nouns that attach to the conjunctive particle て ("te") and are +# semantically verb-like. +# e.g. ごらん, ご覧, 御覧, 頂戴 +#名詞-動詞非自立的 +# +# noun-quotation: text that cannot be segmented into words, proverbs, Chinese poetry, +# dialects, English, etc. Currently, the only entry for 名詞 引用文字列 ("noun quotation") +# is いわく ("iwaku"). +#名詞-引用文字列 +# +# noun-nai_adjective: Words that appear before the auxiliary verb ない ("nai") and +# behave like an adjective. +# e.g. 申し訳, 仕方, とんでも, 違い +#名詞-ナイ形容詞語幹 +# +##### +# prefix: unclassified prefixes +#接頭詞 +# +# prefix-nominal: Prefixes that attach to nouns (including adjective stem forms) +# excluding numerical expressions. +# e.g. お (水), 某 (氏), 同 (社), 故 (～氏), 高 (品質), お (見事), ご (立派) +#接頭詞-名詞接続 +# +# prefix-verbal: Prefixes that attach to the imperative form of a verb or a verb +# in conjunctive form followed by なる/なさる/くださる. +# e.g. お (読みなさい), お (座り) +#接頭詞-動詞接続 +# +# prefix-adjectival: Prefixes that attach to adjectives. +# e.g. お (寒いですねえ), バカ (でかい) +#接頭詞-形容詞接続 +# +# prefix-numerical: Prefixes that attach to numerical expressions. +# e.g. 約, およそ, 毎時 +#接頭詞-数接続 +# +##### +# verb: unclassified verbs +#動詞 +# +# verb-main: +#動詞-自立 +# +# verb-auxiliary: +#動詞-非自立 +# +# verb-suffix: +#動詞-接尾 +# +##### +# adjective: unclassified adjectives +#形容詞 +# +# adjective-main: +#形容詞-自立 +# +# adjective-auxiliary: +#形容詞-非自立 +# +# adjective-suffix: +#形容詞-接尾 +# +##### +# adverb: unclassified adverbs +#副詞 +# +# adverb-misc: Words that can be segmented into one unit and where adnominal +# modification is not possible. +# e.g. あいかわらず, 多分 +#副詞-一般 +# +# adverb-particle_conjunction: Adverbs that can be followed by の, は, に, +# な, する, だ, etc. +# e.g. こんなに, そんなに, あんなに, なにか, なんでも +#副詞-助詞類接続 +# +##### +# adnominal: Words that only have noun-modifying forms. +# e.g. この, その, あの, どの, いわゆる, なんらかの, 何らかの, いろんな, こういう, そういう, ああいう, +# どういう, こんな, そんな, あんな, どんな, 大きな, 小さな, おかしな, ほんの, たいした, +# 「(, も) さる (ことながら)」, 微々たる, 堂々たる, 単なる, いかなる, 我が」「同じ, 亡き +#連体詞 +# +##### +# conjunction: Conjunctions that can occur independently. +# e.g. が, けれども, そして, じゃあ, それどころか +接続詞 +# +##### +# particle: unclassified particles. +助詞 +# +# particle-case: case particles where the subclassification is undefined. +助詞-格助詞 +# +# particle-case-misc: Case particles. +# e.g. から, が, で, と, に, へ, より, を, の, にて +助詞-格助詞-一般 +# +# particle-case-quote: the "to" that appears after nouns, a person’s speech, +# quotation marks, expressions of decisions from a meeting, reasons, judgements, +# conjectures, etc. +# e.g. ( だ) と (述べた.), ( である) と (して執行猶予...) +助詞-格助詞-引用 +# +# particle-case-compound: Compounds of particles and verbs that mainly behave +# like case particles. +# e.g. という, といった, とかいう, として, とともに, と共に, でもって, にあたって, に当たって, に当って, +# にあたり, に当たり, に当り, に当たる, にあたる, において, に於いて,に於て, における, に於ける, +# にかけ, にかけて, にかんし, に関し, にかんして, に関して, にかんする, に関する, に際し, +# に際して, にしたがい, に従い, に従う, にしたがって, に従って, にたいし, に対し, にたいして, +# に対して, にたいする, に対する, について, につき, につけ, につけて, につれ, につれて, にとって, +# にとり, にまつわる, によって, に依って, に因って, により, に依り, に因り, による, に依る, に因る, +# にわたって, にわたる, をもって, を以って, を通じ, を通じて, を通して, をめぐって, をめぐり, をめぐる, +# って-口語/, ちゅう-関西弁「という」/, (何) ていう (人)-口語/, っていう-口語/, といふ, とかいふ +助詞-格助詞-連語 +# +# particle-conjunctive: +# e.g. から, からには, が, けれど, けれども, けど, し, つつ, て, で, と, ところが, どころか, とも, ども, +# ながら, なり, ので, のに, ば, ものの, や ( した), やいなや, (ころん) じゃ(いけない)-口語/, +# (行っ) ちゃ(いけない)-口語/, (言っ) たって (しかたがない)-口語/, (それがなく)ったって (平気)-口語/ +助詞-接続助詞 +# +# particle-dependency: +# e.g. こそ, さえ, しか, すら, は, も, ぞ +助詞-係助詞 +# +# particle-adverbial: +# e.g. がてら, かも, くらい, 位, ぐらい, しも, (学校) じゃ(これが流行っている)-口語/, +# (それ)じゃあ (よくない)-口語/, ずつ, (私) なぞ, など, (私) なり (に), (先生) なんか (大嫌い)-口語/, +# (私) なんぞ, (先生) なんて (大嫌い)-口語/, のみ, だけ, (私) だって-口語/, だに, +# (彼)ったら-口語/, (お茶) でも (いかが), 等 (とう), (今後) とも, ばかり, ばっか-口語/, ばっかり-口語/, +# ほど, 程, まで, 迄, (誰) も (が)([助詞-格助詞] および [助詞-係助詞] の前に位置する「も」) +助詞-副助詞 +# +# particle-interjective: particles with interjective grammatical roles. +# e.g. (松島) や +助詞-間投助詞 +# +# particle-coordinate: +# e.g. と, たり, だの, だり, とか, なり, や, やら +助詞-並立助詞 +# +# particle-final: +# e.g. かい, かしら, さ, ぜ, (だ)っけ-口語/, (とまってる) で-方言/, な, ナ, なあ-口語/, ぞ, ね, ネ, +# ねぇ-口語/, ねえ-口語/, ねん-方言/, の, のう-口語/, や, よ, ヨ, よぉ-口語/, わ, わい-口語/ +助詞-終助詞 +# +# particle-adverbial/conjunctive/final: The particle "ka" when unknown whether it is +# adverbial, conjunctive, or sentence final. For example: +# (a) 「A か B か」. Ex:「(国内で運用する) か,(海外で運用する) か (.)」 +# (b) Inside an adverb phrase. Ex:「(幸いという) か (, 死者はいなかった.)」 +# 「(祈りが届いたせい) か (, 試験に合格した.)」 +# (c) 「かのように」. Ex:「(何もなかった) か (のように振る舞った.)」 +# e.g. か +助詞-副助詞／並立助詞／終助詞 +# +# particle-adnominalizer: The "no" that attaches to nouns and modifies +# non-inflectional words. +助詞-連体化 +# +# particle-adnominalizer: The "ni" and "to" that appear following nouns and adverbs +# that are giongo, giseigo, or gitaigo. +# e.g. に, と +助詞-副詞化 +# +# particle-special: A particle that does not fit into one of the above classifications. +# This includes particles that are used in Tanka, Haiku, and other poetry. +# e.g. かな, けむ, ( しただろう) に, (あんた) にゃ(わからん), (俺) ん (家) +助詞-特殊 +# +##### +# auxiliary-verb: +助動詞 +# +##### +# interjection: Greetings and other exclamations. +# e.g. おはよう, おはようございます, こんにちは, こんばんは, ありがとう, どうもありがとう, ありがとうございます, +# いただきます, ごちそうさま, さよなら, さようなら, はい, いいえ, ごめん, ごめんなさい +#感動詞 +# +##### +# symbol: unclassified Symbols. +記号 +# +# symbol-misc: A general symbol not in one of the categories below. +# e.g. [○◎@$〒→+] +記号-一般 +# +# symbol-comma: Commas +# e.g. [,、] +記号-読点 +# +# symbol-period: Periods and full stops. +# e.g. [.．。] +記号-句点 +# +# symbol-space: Full-width whitespace. +記号-空白 +# +# symbol-open_bracket: +# e.g. [({‘“『【] +記号-括弧開 +# +# symbol-close_bracket: +# e.g. [)}’”』」】] +記号-括弧閉 +# +# symbol-alphabetic: +#記号-アルファベット +# +##### +# other: unclassified other +#その他 +# +# other-interjection: Words that are hard to classify as noun-suffixes or +# sentence-final particles. +# e.g. (だ)ァ +その他-間投 +# +##### +# filler: Aizuchi that occurs during a conversation or sounds inserted as filler. +# e.g. あの, うんと, えと +フィラー +# +##### +# non-verbal: non-verbal sound. +非言語音 +# +##### +# fragment: +#語断片 +# +##### +# unknown: unknown part of speech. +#未知語 +# +##### End of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ar.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ar.txt new file mode 100644 index 000000000..046829db6 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ar.txt @@ -0,0 +1,125 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +# Cleaned on October 11, 2009 (not normalized, so use before normalization) +# This means that when modifying this list, you might need to add some +# redundant entries, for example containing forms with both أ and ا +من +ومن +منها +منه +في +وفي +فيها +فيه +و +ف +ثم +او +أو +ب +بها +به +ا +أ +اى +اي +أي +أى +لا +ولا +الا +ألا +إلا +لكن +ما +وما +كما +فما +عن +مع +اذا +إذا +ان +أن +إن +انها +أنها +إنها +انه +أنه +إنه +بان +بأن +فان +فأن +وان +وأن +وإن +التى +التي +الذى +الذي +الذين +الى +الي +إلى +إلي +على +عليها +عليه +اما +أما +إما +ايضا +أيضا +كل +وكل +لم +ولم +لن +ولن +هى +هي +هو +وهى +وهي +وهو +فهى +فهي +فهو +انت +أنت +لك +لها +له +هذه +هذا +تلك +ذلك +هناك +كانت +كان +يكون +تكون +وكانت +وكان +غير +بعض +قد +نحو +بين +بينما +منذ +ضمن +حيث +الان +الآن +خلال +بعد +قبل +حتى +عند +عندما +لدى +جميع diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_bg.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_bg.txt new file mode 100644 index 000000000..1ae4ba2ae --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_bg.txt @@ -0,0 +1,193 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +а +аз +ако +ала +бе +без +беше +би +бил +била +били +било +близо +бъдат +бъде +бяха +в +вас +ваш +ваша +вероятно +вече +взема +ви +вие +винаги +все +всеки +всички +всичко +всяка +във +въпреки +върху +г +ги +главно +го +д +да +дали +до +докато +докога +дори +досега +доста +е +едва +един +ето +за +зад +заедно +заради +засега +затова +защо +защото +и +из +или +им +има +имат +иска +й +каза +как +каква +какво +както +какъв +като +кога +когато +което +които +кой +който +колко +която +къде +където +към +ли +м +ме +между +мен +ми +мнозина +мога +могат +може +моля +момента +му +н +на +над +назад +най +направи +напред +например +нас +не +него +нея +ни +ние +никой +нито +но +някои +някой +няма +обаче +около +освен +особено +от +отгоре +отново +още +пак +по +повече +повечето +под +поне +поради +после +почти +прави +пред +преди +през +при +пък +първо +с +са +само +се +сега +си +скоро +след +сме +според +сред +срещу +сте +съм +със +също +т +тази +така +такива +такъв +там +твой +те +тези +ти +тн +то +това +тогава +този +той +толкова +точно +трябва +тук +тъй +тя +тях +у +харесва +ч +че +често +чрез +ще +щом +я diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ca.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ca.txt new file mode 100644 index 000000000..3da65deaf --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ca.txt @@ -0,0 +1,220 @@ +# Catalan stopwords from http://github.com/vcl/cue.language (Apache 2 Licensed) +a +abans +ací +ah +així +això +al +als +aleshores +algun +alguna +algunes +alguns +alhora +allà +allí +allò +altra +altre +altres +amb +ambdós +ambdues +apa +aquell +aquella +aquelles +aquells +aquest +aquesta +aquestes +aquests +aquí +baix +cada +cadascú +cadascuna +cadascunes +cadascuns +com +contra +d'un +d'una +d'unes +d'uns +dalt +de +del +dels +des +després +dins +dintre +donat +doncs +durant +e +eh +el +els +em +en +encara +ens +entre +érem +eren +éreu +es +és +esta +està +estàvem +estaven +estàveu +esteu +et +etc +ets +fins +fora +gairebé +ha +han +has +havia +he +hem +heu +hi +ho +i +igual +iguals +ja +l'hi +la +les +li +li'n +llavors +m'he +ma +mal +malgrat +mateix +mateixa +mateixes +mateixos +me +mentre +més +meu +meus +meva +meves +molt +molta +moltes +molts +mon +mons +n'he +n'hi +ne +ni +no +nogensmenys +només +nosaltres +nostra +nostre +nostres +o +oh +oi +on +pas +pel +pels +per +però +perquè +poc +poca +pocs +poques +potser +propi +qual +quals +quan +quant +que +què +quelcom +qui +quin +quina +quines +quins +s'ha +s'han +sa +semblant +semblants +ses +seu +seus +seva +seva +seves +si +sobre +sobretot +sóc +solament +sols +son +són +sons +sota +sou +t'ha +t'han +t'he +ta +tal +també +tampoc +tan +tant +tanta +tantes +teu +teus +teva +teves +ton +tons +tot +tota +totes +tots +un +una +unes +uns +us +va +vaig +vam +van +vas +veu +vosaltres +vostra +vostre +vostres diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ckb.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ckb.txt new file mode 100644 index 000000000..87abf118f --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ckb.txt @@ -0,0 +1,136 @@ +# set of kurdish stopwords +# note these have been normalized with our scheme (e represented with U+06D5, etc) +# constructed from: +# * Fig 5 of "Building A Test Collection For Sorani Kurdish" (Esmaili et al) +# * "Sorani Kurdish: A Reference Grammar with selected readings" (Thackston) +# * Corpus-based analysis of 77M word Sorani collection: wikipedia, news, blogs, etc + +# and +و +# which +کە +# of +ی +# made/did +کرد +# that/which +ئەوەی +# on/head +سەر +# two +دوو +# also +هەروەها +# from/that +لەو +# makes/does +دەکات +# some +چەند +# every +هەر + +# demonstratives +# that +ئەو +# this +ئەم + +# personal pronouns +# I +من +# we +ئێمە +# you +تۆ +# you +ئێوە +# he/she/it +ئەو +# they +ئەوان + +# prepositions +# to/with/by +بە +پێ +# without +بەبێ +# along with/while/during +بەدەم +# in the opinion of +بەلای +# according to +بەپێی +# before +بەرلە +# in the direction of +بەرەوی +# in front of/toward +بەرەوە +# before/in the face of +بەردەم +# without +بێ +# except for +بێجگە +# for +بۆ +# on/in +دە +تێ +# with +دەگەڵ +# after +دوای +# except for/aside from +جگە +# in/from +لە +لێ +# in front of/before/because of +لەبەر +# between/among +لەبەینی +# concerning/about +لەبابەت +# concerning +لەبارەی +# instead of +لەباتی +# beside +لەبن +# instead of +لەبرێتی +# behind +لەدەم +# with/together with +لەگەڵ +# by +لەلایەن +# within +لەناو +# between/among +لەنێو +# for the sake of +لەپێناوی +# with respect to +لەرەوی +# by means of/for +لەرێ +# for the sake of +لەرێگا +# on/on top of/according to +لەسەر +# under +لەژێر +# between/among +ناو +# between/among +نێوان +# after +پاش +# before +پێش +# like +وەک diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_cz.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_cz.txt new file mode 100644 index 000000000..53c6097da --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_cz.txt @@ -0,0 +1,172 @@ +a +s +k +o +i +u +v +z +dnes +cz +tímto +budeš +budem +byli +jseš +můj +svým +ta +tomto +tohle +tuto +tyto +jej +zda +proč +máte +tato +kam +tohoto +kdo +kteří +mi +nám +tom +tomuto +mít +nic +proto +kterou +byla +toho +protože +asi +ho +naši +napište +re +což +tím +takže +svých +její +svými +jste +aj +tu +tedy +teto +bylo +kde +ke +pravé +ji +nad +nejsou +či +pod +téma +mezi +přes +ty +pak +vám +ani +když +však +neg +jsem +tento +článku +články +aby +jsme +před +pta +jejich +byl +ještě +až +bez +také +pouze +první +vaše +která +nás +nový +tipy +pokud +může +strana +jeho +své +jiné +zprávy +nové +není +vás +jen +podle +zde +už +být +více +bude +již +než +který +by +které +co +nebo +ten +tak +má +při +od +po +jsou +jak +další +ale +si +se +ve +to +jako +za +zpět +ze +do +pro +je +na +atd +atp +jakmile +přičemž +já +on +ona +ono +oni +ony +my +vy +jí +ji +mě +mne +jemu +tomu +těm +těmu +němu +němuž +jehož +jíž +jelikož +jež +jakož +načež diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_da.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_da.txt new file mode 100644 index 000000000..42e6145b9 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_da.txt @@ -0,0 +1,110 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/danish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Danish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + +og | and +i | in +jeg | I +det | that (dem. pronoun)/it (pers. pronoun) +at | that (in front of a sentence)/to (with infinitive) +en | a/an +den | it (pers. pronoun)/that (dem. pronoun) +til | to/at/for/until/against/by/of/into, more +er | present tense of "to be" +som | who, as +på | on/upon/in/on/at/to/after/of/with/for, on +de | they +med | with/by/in, along +han | he +af | of/by/from/off/for/in/with/on, off +for | at/for/to/from/by/of/ago, in front/before, because +ikke | not +der | who/which, there/those +var | past tense of "to be" +mig | me/myself +sig | oneself/himself/herself/itself/themselves +men | but +et | a/an/one, one (number), someone/somebody/one +har | present tense of "to have" +om | round/about/for/in/a, about/around/down, if +vi | we +min | my +havde | past tense of "to have" +ham | him +hun | she +nu | now +over | over/above/across/by/beyond/past/on/about, over/past +da | then, when/as/since +fra | from/off/since, off, since +du | you +ud | out +sin | his/her/its/one's +dem | them +os | us/ourselves +op | up +man | you/one +hans | his +hvor | where +eller | or +hvad | what +skal | must/shall etc. +selv | myself/youself/herself/ourselves etc., even +her | here +alle | all/everyone/everybody etc. +vil | will (verb) +blev | past tense of "to stay/to remain/to get/to become" +kunne | could +ind | in +når | when +være | present tense of "to be" +dog | however/yet/after all +noget | something +ville | would +jo | you know/you see (adv), yes +deres | their/theirs +efter | after/behind/according to/for/by/from, later/afterwards +ned | down +skulle | should +denne | this +end | than +dette | this +mit | my/mine +også | also +under | under/beneath/below/during, below/underneath +have | have +dig | you +anden | other +hende | her +mine | my +alt | everything +meget | much/very, plenty of +sit | his, her, its, one's +sine | his, her, its, one's +vor | our +mod | against +disse | these +hvis | if +din | your/yours +nogle | some +hos | by/at +blive | be/become +mange | many +ad | by/through +bliver | present tense of "to be/to become" +hendes | her/hers +været | be +thi | for (conj) +jer | you +sådan | such, like this/like that diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_de.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_de.txt new file mode 100644 index 000000000..86525e7ae --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_de.txt @@ -0,0 +1,294 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A German stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | The number of forms in this list is reduced significantly by passing it + | through the German stemmer. + + +aber | but + +alle | all +allem +allen +aller +alles + +als | than, as +also | so +am | an + dem +an | at + +ander | other +andere +anderem +anderen +anderer +anderes +anderm +andern +anderr +anders + +auch | also +auf | on +aus | out of +bei | by +bin | am +bis | until +bist | art +da | there +damit | with it +dann | then + +der | the +den +des +dem +die +das + +daß | that + +derselbe | the same +derselben +denselben +desselben +demselben +dieselbe +dieselben +dasselbe + +dazu | to that + +dein | thy +deine +deinem +deinen +deiner +deines + +denn | because + +derer | of those +dessen | of him + +dich | thee +dir | to thee +du | thou + +dies | this +diese +diesem +diesen +dieser +dieses + + +doch | (several meanings) +dort | (over) there + + +durch | through + +ein | a +eine +einem +einen +einer +eines + +einig | some +einige +einigem +einigen +einiger +einiges + +einmal | once + +er | he +ihn | him +ihm | to him + +es | it +etwas | something + +euer | your +eure +eurem +euren +eurer +eures + +für | for +gegen | towards +gewesen | p.p. of sein +hab | have +habe | have +haben | have +hat | has +hatte | had +hatten | had +hier | here +hin | there +hinter | behind + +ich | I +mich | me +mir | to me + + +ihr | you, to her +ihre +ihrem +ihren +ihrer +ihres +euch | to you + +im | in + dem +in | in +indem | while +ins | in + das +ist | is + +jede | each, every +jedem +jeden +jeder +jedes + +jene | that +jenem +jenen +jener +jenes + +jetzt | now +kann | can + +kein | no +keine +keinem +keinen +keiner +keines + +können | can +könnte | could +machen | do +man | one + +manche | some, many a +manchem +manchen +mancher +manches + +mein | my +meine +meinem +meinen +meiner +meines + +mit | with +muss | must +musste | had to +nach | to(wards) +nicht | not +nichts | nothing +noch | still, yet +nun | now +nur | only +ob | whether +oder | or +ohne | without +sehr | very + +sein | his +seine +seinem +seinen +seiner +seines + +selbst | self +sich | herself + +sie | they, she +ihnen | to them + +sind | are +so | so + +solche | such +solchem +solchen +solcher +solches + +soll | shall +sollte | should +sondern | but +sonst | else +über | over +um | about, around +und | and + +uns | us +unse +unsem +unsen +unser +unses + +unter | under +viel | much +vom | von + dem +von | from +vor | before +während | while +war | was +waren | were +warst | wast +was | what +weg | away, off +weil | because +weiter | further + +welche | which +welchem +welchen +welcher +welches + +wenn | when +werde | will +werden | will +wie | how +wieder | again +will | want +wir | we +wird | will +wirst | willst +wo | where +wollen | want +wollte | wanted +würde | would +würden | would +zu | to +zum | zu + dem +zur | zu + der +zwar | indeed +zwischen | between + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_el.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_el.txt new file mode 100644 index 000000000..232681f5b --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_el.txt @@ -0,0 +1,78 @@ +# Lucene Greek Stopwords list +# Note: by default this file is used after GreekLowerCaseFilter, +# so when modifying this file use 'σ' instead of 'ς' +ο +η +το +οι +τα +του +τησ +των +τον +την +και +κι +κ +ειμαι +εισαι +ειναι +ειμαστε +ειστε +στο +στον +στη +στην +μα +αλλα +απο +για +προσ +με +σε +ωσ +παρα +αντι +κατα +μετα +θα +να +δε +δεν +μη +μην +επι +ενω +εαν +αν +τοτε +που +πωσ +ποιοσ +ποια +ποιο +ποιοι +ποιεσ +ποιων +ποιουσ +αυτοσ +αυτη +αυτο +αυτοι +αυτων +αυτουσ +αυτεσ +αυτα +εκεινοσ +εκεινη +εκεινο +εκεινοι +εκεινεσ +εκεινα +εκεινων +εκεινουσ +οπωσ +ομωσ +ισωσ +οσο +οτι diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_en.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_en.txt new file mode 100644 index 000000000..8bb2b7de4 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_en.txt @@ -0,0 +1,332 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +### Top 200 words based on frequency in common use + stemmed versions +### > 1000 per million words +the +of +and +a +in +to +it +is +was +wa +I +for +that +you +he +be +with +on +by +at +have +are +ar +not +this +thi +but +had +they +thei +his +hi +from + +### > 3000 PMW +she +which +or +we +an +were +as + +### > 2000 PMW +do +been +their +has +ha +would +there +what +will +all +if +can +her +#said +who + +### Top 50 +### > 1000 PMW +#one +#on +so +up +them +some +when +could +him +into +its +it +then +#two +out +#time +my +about +#did +#your +#now +me +no +other +only +onli +#just +more +these +also +#people +#peopl +#know +any +ani +#first +#see +very +veri +new +#may +#mai +#well +should +#like +than +how + +### Top 90 +### > 900 PMW +#get +#way +#wai +our +#made +#got +after +#think +between +#many +#mani +#years +#year + +### Top 100 +### > 800 PMW +er +those +#go +being +be +because +becaus +down +#yeah + +### > 700 PMW +#three +#good +#back +#make +such +#through +#year +#over +#must +#still +#even +#take +#too + +### Top 120 above + +### > 600 PMW +#here +#come +#own +#last +#does +#doe +#oh +#say +#sai +#work +#where +#erm +#us +#government +#govern +#same +#man +#might +#day +#dai +#yes +#ye +#however +#howev + +### > 500 PMW +#put +#world +#another +#anoth +#want +#life +#most +#against +#again +#never +#under +#old +#much +#something +#someth +#Mr +#why +#each +#while +#house +#hous + +### > 400 PMW +#part +#number +#out of +#found +#off +#different +#differ +#went +#really +#realli +#thought +#came +#used +#us +#children +#always +#alwai +#four +#without +#give +#few +#within +#system +#local +#place +#great +#during +#dure +#although +#small +#before +#befor +#look +#case +#next +#end +#things +#thing +#social +#find +#group +#quite +#quit +#mean +#five +#party +#parti +#company +#compani +#every +#everi + +### > 300 PMW +#women +#says +#sai +#important +#import +#took + +### Top 200 + + + + + + +###################### Lucene defaults .... + + +# Standard english stop words taken from Lucene's StopAnalyzer +# - Added stemmed varients - are -> ar, they -> thei, this -> thi, was -> wa +#a +#an +#and +#are +#ar +#as +#at +#be +#but +#by +#for +#if +#in +#into +#is +#it +#no +#not +#of +#on +#or +#such +#that +#the +#their +#then +#there +#these +#they +#thei +#this +#thi +#to +#was +#wa +#will +#with diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_es.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_es.txt new file mode 100644 index 000000000..487d78c8d --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_es.txt @@ -0,0 +1,356 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Spanish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + + | The following is a ranked list (commonest to rarest) of stopwords + | deriving from a large sample of text. + + | Extra words have been added at the end. + +de | from, of +la | the, her +que | who, that +el | the +en | in +y | and +a | to +los | the, them +del | de + el +se | himself, from him etc +las | the, them +por | for, by, etc +un | a +para | for +con | with +no | no +una | a +su | his, her +al | a + el + | es from SER +lo | him +como | how +más | more +pero | pero +sus | su plural +le | to him, her +ya | already +o | or + | fue from SER +este | this + | ha from HABER +sí | himself etc +porque | because +esta | this + | son from SER +entre | between + | está from ESTAR +cuando | when +muy | very +sin | without +sobre | on + | ser from SER + | tiene from TENER +también | also +me | me +hasta | until +hay | there is/are +donde | where + | han from HABER +quien | whom, that + | están from ESTAR + | estado from ESTAR +desde | from +todo | all +nos | us +durante | during + | estados from ESTAR +todos | all +uno | a +les | to them +ni | nor +contra | against +otros | other + | fueron from SER +ese | that +eso | that + | había from HABER +ante | before +ellos | they +e | and (variant of y) +esto | this +mí | me +antes | before +algunos | some +qué | what? +unos | a +yo | I +otro | other +otras | other +otra | other +él | he +tanto | so much, many +esa | that +estos | these +mucho | much, many +quienes | who +nada | nothing +muchos | many +cual | who + | sea from SER +poco | few +ella | she +estar | to be + | haber from HABER +estas | these + | estaba from ESTAR + | estamos from ESTAR +algunas | some +algo | something +nosotros | we + + | other forms + +mi | me +mis | mi plural +tú | thou +te | thee +ti | thee +tu | thy +tus | tu plural +ellas | they +nosotras | we +vosotros | you +vosotras | you +os | you +mío | mine +mía | +míos | +mías | +tuyo | thine +tuya | +tuyos | +tuyas | +suyo | his, hers, theirs +suya | +suyos | +suyas | +nuestro | ours +nuestra | +nuestros | +nuestras | +vuestro | yours +vuestra | +vuestros | +vuestras | +esos | those +esas | those + + | forms of estar, to be (not including the infinitive): +estoy +estás +está +estamos +estáis +están +esté +estés +estemos +estéis +estén +estaré +estarás +estará +estaremos +estaréis +estarán +estaría +estarías +estaríamos +estaríais +estarían +estaba +estabas +estábamos +estabais +estaban +estuve +estuviste +estuvo +estuvimos +estuvisteis +estuvieron +estuviera +estuvieras +estuviéramos +estuvierais +estuvieran +estuviese +estuvieses +estuviésemos +estuvieseis +estuviesen +estando +estado +estada +estados +estadas +estad + + | forms of haber, to have (not including the infinitive): +he +has +ha +hemos +habéis +han +haya +hayas +hayamos +hayáis +hayan +habré +habrás +habrá +habremos +habréis +habrán +habría +habrías +habríamos +habríais +habrían +había +habías +habíamos +habíais +habían +hube +hubiste +hubo +hubimos +hubisteis +hubieron +hubiera +hubieras +hubiéramos +hubierais +hubieran +hubiese +hubieses +hubiésemos +hubieseis +hubiesen +habiendo +habido +habida +habidos +habidas + + | forms of ser, to be (not including the infinitive): +soy +eres +es +somos +sois +son +sea +seas +seamos +seáis +sean +seré +serás +será +seremos +seréis +serán +sería +serías +seríamos +seríais +serían +era +eras +éramos +erais +eran +fui +fuiste +fue +fuimos +fuisteis +fueron +fuera +fueras +fuéramos +fuerais +fueran +fuese +fueses +fuésemos +fueseis +fuesen +siendo +sido + | sed also means 'thirst' + + | forms of tener, to have (not including the infinitive): +tengo +tienes +tiene +tenemos +tenéis +tienen +tenga +tengas +tengamos +tengáis +tengan +tendré +tendrás +tendrá +tendremos +tendréis +tendrán +tendría +tendrías +tendríamos +tendríais +tendrían +tenía +tenías +teníamos +teníais +tenían +tuve +tuviste +tuvo +tuvimos +tuvisteis +tuvieron +tuviera +tuvieras +tuviéramos +tuvierais +tuvieran +tuviese +tuvieses +tuviésemos +tuvieseis +tuviesen +teniendo +tenido +tenida +tenidos +tenidas +tened + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_eu.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_eu.txt new file mode 100644 index 000000000..25f1db934 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_eu.txt @@ -0,0 +1,99 @@ +# example set of basque stopwords +al +anitz +arabera +asko +baina +bat +batean +batek +bati +batzuei +batzuek +batzuetan +batzuk +bera +beraiek +berau +berauek +bere +berori +beroriek +beste +bezala +da +dago +dira +ditu +du +dute +edo +egin +ere +eta +eurak +ez +gainera +gu +gutxi +guzti +haiei +haiek +haietan +hainbeste +hala +han +handik +hango +hara +hari +hark +hartan +hau +hauei +hauek +hauetan +hemen +hemendik +hemengo +hi +hona +honek +honela +honetan +honi +hor +hori +horiei +horiek +horietan +horko +horra +horrek +horrela +horretan +horri +hortik +hura +izan +ni +noiz +nola +non +nondik +nongo +nor +nora +ze +zein +zen +zenbait +zenbat +zer +zergatik +ziren +zituen +zu +zuek +zuen +zuten diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fa.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fa.txt new file mode 100644 index 000000000..723641c6d --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fa.txt @@ -0,0 +1,313 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +# Note: by default this file is used after normalization, so when adding entries +# to this file, use the arabic 'ي' instead of 'ی' +انان +نداشته +سراسر +خياه +ايشان +وي +تاكنون +بيشتري +دوم +پس +ناشي +وگو +يا +داشتند +سپس +هنگام +هرگز +پنج +نشان +امسال +ديگر +گروهي +شدند +چطور +ده +و +دو +نخستين +ولي +چرا +چه +وسط +ه +كدام +قابل +يك +رفت +هفت +همچنين +در +هزار +بله +بلي +شايد +اما +شناسي +گرفته +دهد +داشته +دانست +داشتن +خواهيم +ميليارد +وقتيكه +امد +خواهد +جز +اورده +شده +بلكه +خدمات +شدن +برخي +نبود +بسياري +جلوگيري +حق +كردند +نوعي +بعري +نكرده +نظير +نبايد +بوده +بودن +داد +اورد +هست +جايي +شود +دنبال +داده +بايد +سابق +هيچ +همان +انجا +كمتر +كجاست +گردد +كسي +تر +مردم +تان +دادن +بودند +سري +جدا +ندارند +مگر +يكديگر +دارد +دهند +بنابراين +هنگامي +سمت +جا +انچه +خود +دادند +زياد +دارند +اثر +بدون +بهترين +بيشتر +البته +به +براساس +بيرون +كرد +بعضي +گرفت +توي +اي +ميليون +او +جريان +تول +بر +مانند +برابر +باشيم +مدتي +گويند +اكنون +تا +تنها +جديد +چند +بي +نشده +كردن +كردم +گويد +كرده +كنيم +نمي +نزد +روي +قصد +فقط +بالاي +ديگران +اين +ديروز +توسط +سوم +ايم +دانند +سوي +استفاده +شما +كنار +داريم +ساخته +طور +امده +رفته +نخست +بيست +نزديك +طي +كنيد +از +انها +تمامي +داشت +يكي +طريق +اش +چيست +روب +نمايد +گفت +چندين +چيزي +تواند +ام +ايا +با +ان +ايد +ترين +اينكه +ديگري +راه +هايي +بروز +همچنان +پاعين +كس +حدود +مختلف +مقابل +چيز +گيرد +ندارد +ضد +همچون +سازي +شان +مورد +باره +مرسي +خويش +برخوردار +چون +خارج +شش +هنوز +تحت +ضمن +هستيم +گفته +فكر +بسيار +پيش +براي +روزهاي +انكه +نخواهد +بالا +كل +وقتي +كي +چنين +كه +گيري +نيست +است +كجا +كند +نيز +يابد +بندي +حتي +توانند +عقب +خواست +كنند +بين +تمام +همه +ما +باشند +مثل +شد +اري +باشد +اره +طبق +بعد +اگر +صورت +غير +جاي +بيش +ريزي +اند +زيرا +چگونه +بار +لطفا +مي +درباره +من +ديده +همين +گذاري +برداري +علت +گذاشته +هم +فوق +نه +ها +شوند +اباد +همواره +هر +اول +خواهند +چهار +نام +امروز +مان +هاي +قبل +كنم +سعي +تازه +را +هستند +زير +جلوي +عنوان +بود diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fi.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fi.txt new file mode 100644 index 000000000..4372c9a05 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fi.txt @@ -0,0 +1,97 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + +| forms of BE + +olla +olen +olet +on +olemme +olette +ovat +ole | negative form + +oli +olisi +olisit +olisin +olisimme +olisitte +olisivat +olit +olin +olimme +olitte +olivat +ollut +olleet + +en | negation +et +ei +emme +ette +eivät + +|Nom Gen Acc Part Iness Elat Illat Adess Ablat Allat Ess Trans +minä minun minut minua minussa minusta minuun minulla minulta minulle | I +sinä sinun sinut sinua sinussa sinusta sinuun sinulla sinulta sinulle | you +hän hänen hänet häntä hänessä hänestä häneen hänellä häneltä hänelle | he she +me meidän meidät meitä meissä meistä meihin meillä meiltä meille | we +te teidän teidät teitä teissä teistä teihin teillä teiltä teille | you +he heidän heidät heitä heissä heistä heihin heillä heiltä heille | they + +tämä tämän tätä tässä tästä tähän tallä tältä tälle tänä täksi | this +tuo tuon tuotä tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that +se sen sitä siinä siitä siihen sillä siltä sille sinä siksi | it +nämä näiden näitä näissä näistä näihin näillä näiltä näille näinä näiksi | these +nuo noiden noita noissa noista noihin noilla noilta noille noina noiksi | those +ne niiden niitä niissä niistä niihin niillä niiltä niille niinä niiksi | they + +kuka kenen kenet ketä kenessä kenestä keneen kenellä keneltä kenelle kenenä keneksi| who +ketkä keiden ketkä keitä keissä keistä keihin keillä keiltä keille keinä keiksi | (pl) +mikä minkä minkä mitä missä mistä mihin millä miltä mille minä miksi | which what +mitkä | (pl) + +joka jonka jota jossa josta johon jolla jolta jolle jona joksi | who which +jotka joiden joita joissa joista joihin joilla joilta joille joina joiksi | (pl) + +| conjunctions + +että | that +ja | and +jos | if +koska | because +kuin | than +mutta | but +niin | so +sekä | and +sillä | for +tai | or +vaan | but +vai | or +vaikka | although + + +| prepositions + +kanssa | with +mukaan | according to +noin | about +poikki | across +yli | over, across + +| other + +kun | when +niin | so +nyt | now +itse | self + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fr.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fr.txt new file mode 100644 index 000000000..749abae68 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fr.txt @@ -0,0 +1,186 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A French stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + +au | a + le +aux | a + les +avec | with +ce | this +ces | these +dans | with +de | of +des | de + les +du | de + le +elle | she +en | `of them' etc +et | and +eux | them +il | he +je | I +la | the +le | the +leur | their +lui | him +ma | my (fem) +mais | but +me | me +même | same; as in moi-même (myself) etc +mes | me (pl) +moi | me +mon | my (masc) +ne | not +nos | our (pl) +notre | our +nous | we +on | one +ou | where +par | by +pas | not +pour | for +qu | que before vowel +que | that +qui | who +sa | his, her (fem) +se | oneself +ses | his (pl) +son | his, her (masc) +sur | on +ta | thy (fem) +te | thee +tes | thy (pl) +toi | thee +ton | thy (masc) +tu | thou +un | a +une | a +vos | your (pl) +votre | your +vous | you + + | single letter forms + +c | c' +d | d' +j | j' +l | l' +à | to, at +m | m' +n | n' +s | s' +t | t' +y | there + + | forms of être (not including the infinitive): +été +étée +étées +étés +étant +suis +es +est +sommes +êtes +sont +serai +seras +sera +serons +serez +seront +serais +serait +serions +seriez +seraient +étais +était +étions +étiez +étaient +fus +fut +fûmes +fûtes +furent +sois +soit +soyons +soyez +soient +fusse +fusses +fût +fussions +fussiez +fussent + + | forms of avoir (not including the infinitive): +ayant +eu +eue +eues +eus +ai +as +avons +avez +ont +aurai +auras +aura +aurons +aurez +auront +aurais +aurait +aurions +auriez +auraient +avais +avait +avions +aviez +avaient +eut +eûmes +eûtes +eurent +aie +aies +ait +ayons +ayez +aient +eusse +eusses +eût +eussions +eussiez +eussent + + | Later additions (from Jean-Christophe Deschamps) +ceci | this +cela | that +celà | that +cet | this +cette | this +ici | here +ils | they +les | the (pl) +leurs | their (pl) +quel | which +quels | which +quelle | which +quelles | which +sans | without +soi | oneself + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ga.txt new file mode 100644 index 000000000..9ff88d747 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ga.txt @@ -0,0 +1,110 @@ + +a +ach +ag +agus +an +aon +ar +arna +as +b' +ba +beirt +bhúr +caoga +ceathair +ceathrar +chomh +chtó +chuig +chun +cois +céad +cúig +cúigear +d' +daichead +dar +de +deich +deichniúr +den +dhá +do +don +dtí +dá +dár +dó +faoi +faoin +faoina +faoinár +fara +fiche +gach +gan +go +gur +haon +hocht +i +iad +idir +in +ina +ins +inár +is +le +leis +lena +lenár +m' +mar +mo +mé +na +nach +naoi +naonúr +ná +ní +níor +nó +nócha +ocht +ochtar +os +roimh +sa +seacht +seachtar +seachtó +seasca +seisear +siad +sibh +sinn +sna +sé +sí +tar +thar +thú +triúr +trí +trína +trínár +tríocha +tú +um +ár +é +éis +í +ó +ón +óna +ónár diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_gl.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_gl.txt new file mode 100644 index 000000000..d8760b12c --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_gl.txt @@ -0,0 +1,161 @@ +# galican stopwords +a +aínda +alí +aquel +aquela +aquelas +aqueles +aquilo +aquí +ao +aos +as +así +á +ben +cando +che +co +coa +comigo +con +connosco +contigo +convosco +coas +cos +cun +cuns +cunha +cunhas +da +dalgunha +dalgunhas +dalgún +dalgúns +das +de +del +dela +delas +deles +desde +deste +do +dos +dun +duns +dunha +dunhas +e +el +ela +elas +eles +en +era +eran +esa +esas +ese +eses +esta +estar +estaba +está +están +este +estes +estiven +estou +eu +é +facer +foi +foron +fun +había +hai +iso +isto +la +las +lle +lles +lo +los +mais +me +meu +meus +min +miña +miñas +moi +na +nas +neste +nin +no +non +nos +nosa +nosas +noso +nosos +nós +nun +nunha +nuns +nunhas +o +os +ou +ó +ós +para +pero +pode +pois +pola +polas +polo +polos +por +que +se +senón +ser +seu +seus +sexa +sido +sobre +súa +súas +tamén +tan +te +ten +teñen +teño +ter +teu +teus +ti +tido +tiña +tiven +túa +túas +un +unha +unhas +uns +vos +vosa +vosas +voso +vosos +vós diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hi.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hi.txt new file mode 100644 index 000000000..86286bb08 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hi.txt @@ -0,0 +1,235 @@ +# Also see http://www.opensource.org/licenses/bsd-license.html +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# This file was created by Jacques Savoy and is distributed under the BSD license. +# Note: by default this file also contains forms normalized by HindiNormalizer +# for spelling variation (see section below), such that it can be used whether or +# not you enable that feature. When adding additional entries to this list, +# please add the normalized form as well. +अंदर +अत +अपना +अपनी +अपने +अभी +आदि +आप +इत्यादि +इन +इनका +इन्हीं +इन्हें +इन्हों +इस +इसका +इसकी +इसके +इसमें +इसी +इसे +उन +उनका +उनकी +उनके +उनको +उन्हीं +उन्हें +उन्हों +उस +उसके +उसी +उसे +एक +एवं +एस +ऐसे +और +कई +कर +करता +करते +करना +करने +करें +कहते +कहा +का +काफ़ी +कि +कितना +किन्हें +किन्हों +किया +किर +किस +किसी +किसे +की +कुछ +कुल +के +को +कोई +कौन +कौनसा +गया +घर +जब +जहाँ +जा +जितना +जिन +जिन्हें +जिन्हों +जिस +जिसे +जीधर +जैसा +जैसे +जो +तक +तब +तरह +तिन +तिन्हें +तिन्हों +तिस +तिसे +तो +था +थी +थे +दबारा +दिया +दुसरा +दूसरे +दो +द्वारा +न +नहीं +ना +निहायत +नीचे +ने +पर +पर +पहले +पूरा +पे +फिर +बनी +बही +बहुत +बाद +बाला +बिलकुल +भी +भीतर +मगर +मानो +मे +में +यदि +यह +यहाँ +यही +या +यिह +ये +रखें +रहा +रहे +ऱ्वासा +लिए +लिये +लेकिन +व +वर्ग +वह +वह +वहाँ +वहीं +वाले +वुह +वे +वग़ैरह +संग +सकता +सकते +सबसे +सभी +साथ +साबुत +साभ +सारा +से +सो +ही +हुआ +हुई +हुए +है +हैं +हो +होता +होती +होते +होना +होने +# additional normalized forms of the above +अपनि +जेसे +होति +सभि +तिंहों +इंहों +दवारा +इसि +किंहें +थि +उंहों +ओर +जिंहें +वहिं +अभि +बनि +हि +उंहिं +उंहें +हें +वगेरह +एसे +रवासा +कोन +निचे +काफि +उसि +पुरा +भितर +हे +बहि +वहां +कोइ +यहां +जिंहों +तिंहें +किसि +कइ +यहि +इंहिं +जिधर +इंहें +अदि +इतयादि +हुइ +कोनसा +इसकि +दुसरे +जहां +अप +किंहों +उनकि +भि +वरग +हुअ +जेसा +नहिं diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hu.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hu.txt new file mode 100644 index 000000000..37526da8a --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hu.txt @@ -0,0 +1,211 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/hungarian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + +| Hungarian stop word list +| prepared by Anna Tordai + +a +ahogy +ahol +aki +akik +akkor +alatt +által +általában +amely +amelyek +amelyekben +amelyeket +amelyet +amelynek +ami +amit +amolyan +amíg +amikor +át +abban +ahhoz +annak +arra +arról +az +azok +azon +azt +azzal +azért +aztán +azután +azonban +bár +be +belül +benne +cikk +cikkek +cikkeket +csak +de +e +eddig +egész +egy +egyes +egyetlen +egyéb +egyik +egyre +ekkor +el +elég +ellen +elő +először +előtt +első +én +éppen +ebben +ehhez +emilyen +ennek +erre +ez +ezt +ezek +ezen +ezzel +ezért +és +fel +felé +hanem +hiszen +hogy +hogyan +igen +így +illetve +ill. +ill +ilyen +ilyenkor +ison +ismét +itt +jó +jól +jobban +kell +kellett +keresztül +keressünk +ki +kívül +között +közül +legalább +lehet +lehetett +legyen +lenne +lenni +lesz +lett +maga +magát +majd +majd +már +más +másik +meg +még +mellett +mert +mely +melyek +mi +mit +míg +miért +milyen +mikor +minden +mindent +mindenki +mindig +mint +mintha +mivel +most +nagy +nagyobb +nagyon +ne +néha +nekem +neki +nem +néhány +nélkül +nincs +olyan +ott +össze +ő +ők +őket +pedig +persze +rá +s +saját +sem +semmi +sok +sokat +sokkal +számára +szemben +szerint +szinte +talán +tehát +teljes +tovább +továbbá +több +úgy +ugyanis +új +újabb +újra +után +utána +utolsó +vagy +vagyis +valaki +valami +valamint +való +vagyok +van +vannak +volt +voltam +voltak +voltunk +vissza +vele +viszont +volna diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hy.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hy.txt new file mode 100644 index 000000000..60c1c50fb --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hy.txt @@ -0,0 +1,46 @@ +# example set of Armenian stopwords. +այդ +այլ +այն +այս +դու +դուք +եմ +են +ենք +ես +եք +է +էի +էին +էինք +էիր +էիք +էր +ըստ +թ +ի +ին +իսկ +իր +կամ +համար +հետ +հետո +մենք +մեջ +մի +ն +նա +նաև +նրա +նրանք +որ +որը +որոնք +որպես +ու +ում +պիտի +վրա +և diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_id.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_id.txt new file mode 100644 index 000000000..4617f83a5 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_id.txt @@ -0,0 +1,359 @@ +# from appendix D of: A Study of Stemming Effects on Information +# Retrieval in Bahasa Indonesia +ada +adanya +adalah +adapun +agak +agaknya +agar +akan +akankah +akhirnya +aku +akulah +amat +amatlah +anda +andalah +antar +diantaranya +antara +antaranya +diantara +apa +apaan +mengapa +apabila +apakah +apalagi +apatah +atau +ataukah +ataupun +bagai +bagaikan +sebagai +sebagainya +bagaimana +bagaimanapun +sebagaimana +bagaimanakah +bagi +bahkan +bahwa +bahwasanya +sebaliknya +banyak +sebanyak +beberapa +seberapa +begini +beginian +beginikah +beginilah +sebegini +begitu +begitukah +begitulah +begitupun +sebegitu +belum +belumlah +sebelum +sebelumnya +sebenarnya +berapa +berapakah +berapalah +berapapun +betulkah +sebetulnya +biasa +biasanya +bila +bilakah +bisa +bisakah +sebisanya +boleh +bolehkah +bolehlah +buat +bukan +bukankah +bukanlah +bukannya +cuma +percuma +dahulu +dalam +dan +dapat +dari +daripada +dekat +demi +demikian +demikianlah +sedemikian +dengan +depan +di +dia +dialah +dini +diri +dirinya +terdiri +dong +dulu +enggak +enggaknya +entah +entahlah +terhadap +terhadapnya +hal +hampir +hanya +hanyalah +harus +haruslah +harusnya +seharusnya +hendak +hendaklah +hendaknya +hingga +sehingga +ia +ialah +ibarat +ingin +inginkah +inginkan +ini +inikah +inilah +itu +itukah +itulah +jangan +jangankan +janganlah +jika +jikalau +juga +justru +kala +kalau +kalaulah +kalaupun +kalian +kami +kamilah +kamu +kamulah +kan +kapan +kapankah +kapanpun +dikarenakan +karena +karenanya +ke +kecil +kemudian +kenapa +kepada +kepadanya +ketika +seketika +khususnya +kini +kinilah +kiranya +sekiranya +kita +kitalah +kok +lagi +lagian +selagi +lah +lain +lainnya +melainkan +selaku +lalu +melalui +terlalu +lama +lamanya +selama +selama +selamanya +lebih +terlebih +bermacam +macam +semacam +maka +makanya +makin +malah +malahan +mampu +mampukah +mana +manakala +manalagi +masih +masihkah +semasih +masing +mau +maupun +semaunya +memang +mereka +merekalah +meski +meskipun +semula +mungkin +mungkinkah +nah +namun +nanti +nantinya +nyaris +oleh +olehnya +seorang +seseorang +pada +padanya +padahal +paling +sepanjang +pantas +sepantasnya +sepantasnyalah +para +pasti +pastilah +per +pernah +pula +pun +merupakan +rupanya +serupa +saat +saatnya +sesaat +saja +sajalah +saling +bersama +sama +sesama +sambil +sampai +sana +sangat +sangatlah +saya +sayalah +se +sebab +sebabnya +sebuah +tersebut +tersebutlah +sedang +sedangkan +sedikit +sedikitnya +segala +segalanya +segera +sesegera +sejak +sejenak +sekali +sekalian +sekalipun +sesekali +sekaligus +sekarang +sekarang +sekitar +sekitarnya +sela +selain +selalu +seluruh +seluruhnya +semakin +sementara +sempat +semua +semuanya +sendiri +sendirinya +seolah +seperti +sepertinya +sering +seringnya +serta +siapa +siapakah +siapapun +disini +disinilah +sini +sinilah +sesuatu +sesuatunya +suatu +sesudah +sesudahnya +sudah +sudahkah +sudahlah +supaya +tadi +tadinya +tak +tanpa +setelah +telah +tentang +tentu +tentulah +tentunya +tertentu +seterusnya +tapi +tetapi +setiap +tiap +setidaknya +tidak +tidakkah +tidaklah +toh +waduh +wah +wahai +sewaktu +walau +walaupun +wong +yaitu +yakni +yang diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_it.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_it.txt new file mode 100644 index 000000000..1219cc773 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_it.txt @@ -0,0 +1,303 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | An Italian stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + +ad | a (to) before vowel +al | a + il +allo | a + lo +ai | a + i +agli | a + gli +all | a + l' +agl | a + gl' +alla | a + la +alle | a + le +con | with +col | con + il +coi | con + i (forms collo, cogli etc are now very rare) +da | from +dal | da + il +dallo | da + lo +dai | da + i +dagli | da + gli +dall | da + l' +dagl | da + gll' +dalla | da + la +dalle | da + le +di | of +del | di + il +dello | di + lo +dei | di + i +degli | di + gli +dell | di + l' +degl | di + gl' +della | di + la +delle | di + le +in | in +nel | in + el +nello | in + lo +nei | in + i +negli | in + gli +nell | in + l' +negl | in + gl' +nella | in + la +nelle | in + le +su | on +sul | su + il +sullo | su + lo +sui | su + i +sugli | su + gli +sull | su + l' +sugl | su + gl' +sulla | su + la +sulle | su + le +per | through, by +tra | among +contro | against +io | I +tu | thou +lui | he +lei | she +noi | we +voi | you +loro | they +mio | my +mia | +miei | +mie | +tuo | +tua | +tuoi | thy +tue | +suo | +sua | +suoi | his, her +sue | +nostro | our +nostra | +nostri | +nostre | +vostro | your +vostra | +vostri | +vostre | +mi | me +ti | thee +ci | us, there +vi | you, there +lo | him, the +la | her, the +li | them +le | them, the +gli | to him, the +ne | from there etc +il | the +un | a +uno | a +una | a +ma | but +ed | and +se | if +perché | why, because +anche | also +come | how +dov | where (as dov') +dove | where +che | who, that +chi | who +cui | whom +non | not +più | more +quale | who, that +quanto | how much +quanti | +quanta | +quante | +quello | that +quelli | +quella | +quelle | +questo | this +questi | +questa | +queste | +si | yes +tutto | all +tutti | all + + | single letter forms: + +a | at +c | as c' for ce or ci +e | and +i | the +l | as l' +o | or + + | forms of avere, to have (not including the infinitive): + +ho +hai +ha +abbiamo +avete +hanno +abbia +abbiate +abbiano +avrò +avrai +avrà +avremo +avrete +avranno +avrei +avresti +avrebbe +avremmo +avreste +avrebbero +avevo +avevi +aveva +avevamo +avevate +avevano +ebbi +avesti +ebbe +avemmo +aveste +ebbero +avessi +avesse +avessimo +avessero +avendo +avuto +avuta +avuti +avute + + | forms of essere, to be (not including the infinitive): +sono +sei +è +siamo +siete +sia +siate +siano +sarò +sarai +sarà +saremo +sarete +saranno +sarei +saresti +sarebbe +saremmo +sareste +sarebbero +ero +eri +era +eravamo +eravate +erano +fui +fosti +fu +fummo +foste +furono +fossi +fosse +fossimo +fossero +essendo + + | forms of fare, to do (not including the infinitive, fa, fat-): +faccio +fai +facciamo +fanno +faccia +facciate +facciano +farò +farai +farà +faremo +farete +faranno +farei +faresti +farebbe +faremmo +fareste +farebbero +facevo +facevi +faceva +facevamo +facevate +facevano +feci +facesti +fece +facemmo +faceste +fecero +facessi +facesse +facessimo +facessero +facendo + + | forms of stare, to be (not including the infinitive): +sto +stai +sta +stiamo +stanno +stia +stiate +stiano +starò +starai +starà +staremo +starete +staranno +starei +staresti +starebbe +staremmo +stareste +starebbero +stavo +stavi +stava +stavamo +stavate +stavano +stetti +stesti +stette +stemmo +steste +stettero +stessi +stesse +stessimo +stessero +stando diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ja.txt new file mode 100644 index 000000000..d4321be6b --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ja.txt @@ -0,0 +1,127 @@ +# +# This file defines a stopword set for Japanese. +# +# This set is made up of hand-picked frequent terms from segmented Japanese Wikipedia. +# Punctuation characters and frequent kanji have mostly been left out. See LUCENE-3745 +# for frequency lists, etc. that can be useful for making your own set (if desired) +# +# Note that there is an overlap between these stopwords and the terms stopped when used +# in combination with the JapanesePartOfSpeechStopFilter. When editing this file, note +# that comments are not allowed on the same line as stopwords. +# +# Also note that stopping is done in a case-insensitive manner. Change your StopFilter +# configuration if you need case-sensitive stopping. Lastly, note that stopping is done +# using the same character width as the entries in this file. Since this StopFilter is +# normally done after a CJKWidthFilter in your chain, you would usually want your romaji +# entries to be in half-width and your kana entries to be in full-width. +# +の +に +は +を +た +が +で +て +と +し +れ +さ +ある +いる +も +する +から +な +こと +として +い +や +れる +など +なっ +ない +この +ため +その +あっ +よう +また +もの +という +あり +まで +られ +なる +へ +か +だ +これ +によって +により +おり +より +による +ず +なり +られる +において +ば +なかっ +なく +しかし +について +せ +だっ +その後 +できる +それ +う +ので +なお +のみ +でき +き +つ +における +および +いう +さらに +でも +ら +たり +その他 +に関する +たち +ます +ん +なら +に対して +特に +せる +及び +これら +とき +では +にて +ほか +ながら +うち +そして +とともに +ただし +かつて +それぞれ +または +お +ほど +ものの +に対する +ほとんど +と共に +といった +です +とも +ところ +ここ +##### End of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_lv.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_lv.txt new file mode 100644 index 000000000..e21a23c06 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_lv.txt @@ -0,0 +1,172 @@ +# Set of Latvian stopwords from A Stemming Algorithm for Latvian, Karlis Kreslins +# the original list of over 800 forms was refined: +# pronouns, adverbs, interjections were removed +# +# prepositions +aiz +ap +ar +apakš +ārpus +augšpus +bez +caur +dēļ +gar +iekš +iz +kopš +labad +lejpus +līdz +no +otrpus +pa +par +pār +pēc +pie +pirms +pret +priekš +starp +šaipus +uz +viņpus +virs +virspus +zem +apakšpus +# Conjunctions +un +bet +jo +ja +ka +lai +tomēr +tikko +turpretī +arī +kaut +gan +tādēļ +tā +ne +tikvien +vien +kā +ir +te +vai +kamēr +# Particles +ar +diezin +droši +diemžēl +nebūt +ik +it +taču +nu +pat +tiklab +iekšpus +nedz +tik +nevis +turpretim +jeb +iekam +iekām +iekāms +kolīdz +līdzko +tiklīdz +jebšu +tālab +tāpēc +nekā +itin +jā +jau +jel +nē +nezin +tad +tikai +vis +tak +iekams +vien +# modal verbs +būt +biju +biji +bija +bijām +bijāt +esmu +esi +esam +esat +būšu +būsi +būs +būsim +būsiet +tikt +tiku +tiki +tika +tikām +tikāt +tieku +tiec +tiek +tiekam +tiekat +tikšu +tiks +tiksim +tiksiet +tapt +tapi +tapāt +topat +tapšu +tapsi +taps +tapsim +tapsiet +kļūt +kļuvu +kļuvi +kļuva +kļuvām +kļuvāt +kļūstu +kļūsti +kļūst +kļūstam +kļūstat +kļūšu +kļūsi +kļūs +kļūsim +kļūsiet +# verbs +varēt +varēju +varējām +varēšu +varēsim +var +varēji +varējāt +varēsi +varēsiet +varat +varēja +varēs diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_nl.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_nl.txt new file mode 100644 index 000000000..47a2aeacf --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_nl.txt @@ -0,0 +1,119 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Dutch stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large sample of Dutch text. + + | Dutch stop words frequently exhibit homonym clashes. These are indicated + | clearly below. + +de | the +en | and +van | of, from +ik | I, the ego +te | (1) chez, at etc, (2) to, (3) too +dat | that, which +die | that, those, who, which +in | in, inside +een | a, an, one +hij | he +het | the, it +niet | not, nothing, naught +zijn | (1) to be, being, (2) his, one's, its +is | is +was | (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river +op | on, upon, at, in, up, used up +aan | on, upon, to (as dative) +met | with, by +als | like, such as, when +voor | (1) before, in front of, (2) furrow +had | had, past tense all persons sing. of 'hebben' (have) +er | there +maar | but, only +om | round, about, for etc +hem | him +dan | then +zou | should/would, past tense all persons sing. of 'zullen' +of | or, whether, if +wat | what, something, anything +mijn | possessive and noun 'mine' +men | people, 'one' +dit | this +zo | so, thus, in this way +door | through by +over | over, across +ze | she, her, they, them +zich | oneself +bij | (1) a bee, (2) by, near, at +ook | also, too +tot | till, until +je | you +mij | me +uit | out of, from +der | Old Dutch form of 'van der' still found in surnames +daar | (1) there, (2) because +haar | (1) her, their, them, (2) hair +naar | (1) unpleasant, unwell etc, (2) towards, (3) as +heb | present first person sing. of 'to have' +hoe | how, why +heeft | present third person sing. of 'to have' +hebben | 'to have' and various parts thereof +deze | this +u | you +want | (1) for, (2) mitten, (3) rigging +nog | yet, still +zal | 'shall', first and third person sing. of verb 'zullen' (will) +me | me +zij | she, they +nu | now +ge | 'thou', still used in Belgium and south Netherlands +geen | none +omdat | because +iets | something, somewhat +worden | to become, grow, get +toch | yet, still +al | all, every, each +waren | (1) 'were' (2) to wander, (3) wares, (3) +veel | much, many +meer | (1) more, (2) lake +doen | to do, to make +toen | then, when +moet | noun 'spot/mote' and present form of 'to must' +ben | (1) am, (2) 'are' in interrogative second person singular of 'to be' +zonder | without +kan | noun 'can' and present form of 'to be able' +hun | their, them +dus | so, consequently +alles | all, everything, anything +onder | under, beneath +ja | yes, of course +eens | once, one day +hier | here +wie | who +werd | imperfect third person sing. of 'become' +altijd | always +doch | yet, but etc +wordt | present third person sing. of 'become' +wezen | (1) to be, (2) 'been' as in 'been fishing', (3) orphans +kunnen | to be able +ons | us/our +zelf | self +tegen | against, towards, at +na | after, near +reeds | already +wil | (1) present tense of 'want', (2) 'will', noun, (3) fender +kon | could; past tense of 'to be able' +niets | nothing +uw | your +iemand | somebody +geweest | been; past participle of 'be' +andere | other diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_no.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_no.txt new file mode 100644 index 000000000..a7a2c28ba --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_no.txt @@ -0,0 +1,194 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/norwegian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Norwegian stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This stop word list is for the dominant bokmål dialect. Words unique + | to nynorsk are marked *. + + | Revised by Jan Bruusgaard , Jan 2005 + +og | and +i | in +jeg | I +det | it/this/that +at | to (w. inf.) +en | a/an +et | a/an +den | it/this/that +til | to +er | is/am/are +som | who/that +på | on +de | they / you(formal) +med | with +han | he +av | of +ikke | not +ikkje | not * +der | there +så | so +var | was/were +meg | me +seg | you +men | but +ett | one +har | have +om | about +vi | we +min | my +mitt | my +ha | have +hadde | had +hun | she +nå | now +over | over +da | when/as +ved | by/know +fra | from +du | you +ut | out +sin | your +dem | them +oss | us +opp | up +man | you/one +kan | can +hans | his +hvor | where +eller | or +hva | what +skal | shall/must +selv | self (reflective) +sjøl | self (reflective) +her | here +alle | all +vil | will +bli | become +ble | became +blei | became * +blitt | have become +kunne | could +inn | in +når | when +være | be +kom | come +noen | some +noe | some +ville | would +dere | you +som | who/which/that +deres | their/theirs +kun | only/just +ja | yes +etter | after +ned | down +skulle | should +denne | this +for | for/because +deg | you +si | hers/his +sine | hers/his +sitt | hers/his +mot | against +å | to +meget | much +hvorfor | why +dette | this +disse | these/those +uten | without +hvordan | how +ingen | none +din | your +ditt | your +blir | become +samme | same +hvilken | which +hvilke | which (plural) +sånn | such a +inni | inside/within +mellom | between +vår | our +hver | each +hvem | who +vors | us/ours +hvis | whose +både | both +bare | only/just +enn | than +fordi | as/because +før | before +mange | many +også | also +slik | just +vært | been +være | to be +båe | both * +begge | both +siden | since +dykk | your * +dykkar | yours * +dei | they * +deira | them * +deires | theirs * +deim | them * +di | your (fem.) * +då | as/when * +eg | I * +ein | a/an * +eit | a/an * +eitt | a/an * +elles | or * +honom | he * +hjå | at * +ho | she * +hoe | she * +henne | her +hennar | her/hers +hennes | hers +hoss | how * +hossen | how * +ikkje | not * +ingi | noone * +inkje | noone * +korleis | how * +korso | how * +kva | what/which * +kvar | where * +kvarhelst | where * +kven | who/whom * +kvi | why * +kvifor | why * +me | we * +medan | while * +mi | my * +mine | my * +mykje | much * +no | now * +nokon | some (masc./neut.) * +noka | some (fem.) * +nokor | some * +noko | some * +nokre | some * +si | his/hers * +sia | since * +sidan | since * +so | so * +somt | some * +somme | some * +um | about* +upp | up * +vere | be * +vore | was * +verte | become * +vort | become * +varte | became * +vart | became * + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_pt.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_pt.txt new file mode 100644 index 000000000..acfeb01af --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_pt.txt @@ -0,0 +1,253 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Portuguese stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + + | The following is a ranked list (commonest to rarest) of stopwords + | deriving from a large sample of text. + + | Extra words have been added at the end. + +de | of, from +a | the; to, at; her +o | the; him +que | who, that +e | and +do | de + o +da | de + a +em | in +um | a +para | for + | é from SER +com | with +não | not, no +uma | a +os | the; them +no | em + o +se | himself etc +na | em + a +por | for +mais | more +as | the; them +dos | de + os +como | as, like +mas | but + | foi from SER +ao | a + o +ele | he +das | de + as + | tem from TER +à | a + a +seu | his +sua | her +ou | or + | ser from SER +quando | when +muito | much + | há from HAV +nos | em + os; us +já | already, now + | está from EST +eu | I +também | also +só | only, just +pelo | per + o +pela | per + a +até | up to +isso | that +ela | he +entre | between + | era from SER +depois | after +sem | without +mesmo | same +aos | a + os + | ter from TER +seus | his +quem | whom +nas | em + as +me | me +esse | that +eles | they + | estão from EST +você | you + | tinha from TER + | foram from SER +essa | that +num | em + um +nem | nor +suas | her +meu | my +às | a + as +minha | my + | têm from TER +numa | em + uma +pelos | per + os +elas | they + | havia from HAV + | seja from SER +qual | which + | será from SER +nós | we + | tenho from TER +lhe | to him, her +deles | of them +essas | those +esses | those +pelas | per + as +este | this + | fosse from SER +dele | of him + + | other words. There are many contractions such as naquele = em+aquele, + | mo = me+o, but they are rare. + | Indefinite article plural forms are also rare. + +tu | thou +te | thee +vocês | you (plural) +vos | you +lhes | to them +meus | my +minhas +teu | thy +tua +teus +tuas +nosso | our +nossa +nossos +nossas + +dela | of her +delas | of them + +esta | this +estes | these +estas | these +aquele | that +aquela | that +aqueles | those +aquelas | those +isto | this +aquilo | that + + | forms of estar, to be (not including the infinitive): +estou +está +estamos +estão +estive +esteve +estivemos +estiveram +estava +estávamos +estavam +estivera +estivéramos +esteja +estejamos +estejam +estivesse +estivéssemos +estivessem +estiver +estivermos +estiverem + + | forms of haver, to have (not including the infinitive): +hei +há +havemos +hão +houve +houvemos +houveram +houvera +houvéramos +haja +hajamos +hajam +houvesse +houvéssemos +houvessem +houver +houvermos +houverem +houverei +houverá +houveremos +houverão +houveria +houveríamos +houveriam + + | forms of ser, to be (not including the infinitive): +sou +somos +são +era +éramos +eram +fui +foi +fomos +foram +fora +fôramos +seja +sejamos +sejam +fosse +fôssemos +fossem +for +formos +forem +serei +será +seremos +serão +seria +seríamos +seriam + + | forms of ter, to have (not including the infinitive): +tenho +tem +temos +tém +tinha +tínhamos +tinham +tive +teve +tivemos +tiveram +tivera +tivéramos +tenha +tenhamos +tenham +tivesse +tivéssemos +tivessem +tiver +tivermos +tiverem +terei +terá +teremos +terão +teria +teríamos +teriam diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ro.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ro.txt new file mode 100644 index 000000000..4fdee90a5 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ro.txt @@ -0,0 +1,233 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +acea +aceasta +această +aceea +acei +aceia +acel +acela +acele +acelea +acest +acesta +aceste +acestea +aceşti +aceştia +acolo +acum +ai +aia +aibă +aici +al +ăla +ale +alea +ălea +altceva +altcineva +am +ar +are +aş +aşadar +asemenea +asta +ăsta +astăzi +astea +ăstea +ăştia +asupra +aţi +au +avea +avem +aveţi +azi +bine +bucur +bună +ca +că +căci +când +care +cărei +căror +cărui +cât +câte +câţi +către +câtva +ce +cel +ceva +chiar +cînd +cine +cineva +cît +cîte +cîţi +cîtva +contra +cu +cum +cumva +curând +curînd +da +dă +dacă +dar +datorită +de +deci +deja +deoarece +departe +deşi +din +dinaintea +dintr +dintre +drept +după +ea +ei +el +ele +eram +este +eşti +eu +face +fără +fi +fie +fiecare +fii +fim +fiţi +iar +ieri +îi +îl +îmi +împotriva +în +înainte +înaintea +încât +încît +încotro +între +întrucât +întrucît +îţi +la +lângă +le +li +lîngă +lor +lui +mă +mâine +mea +mei +mele +mereu +meu +mi +mine +mult +multă +mulţi +ne +nicăieri +nici +nimeni +nişte +noastră +noastre +noi +noştri +nostru +nu +ori +oricând +oricare +oricât +orice +oricînd +oricine +oricît +oricum +oriunde +până +pe +pentru +peste +pînă +poate +pot +prea +prima +primul +prin +printr +sa +să +săi +sale +sau +său +se +şi +sînt +sîntem +sînteţi +spre +sub +sunt +suntem +sunteţi +ta +tăi +tale +tău +te +ţi +ţie +tine +toată +toate +tot +toţi +totuşi +tu +un +una +unde +undeva +unei +unele +uneori +unor +vă +vi +voastră +voastre +voi +voştri +vostru +vouă +vreo +vreun diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ru.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ru.txt new file mode 100644 index 000000000..55271400c --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ru.txt @@ -0,0 +1,243 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/russian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | a russian stop word list. comments begin with vertical bar. each stop + | word is at the start of a line. + + | this is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + | letter `ё' is translated to `е'. + +и | and +в | in/into +во | alternative form +не | not +что | what/that +он | he +на | on/onto +я | i +с | from +со | alternative form +как | how +а | milder form of `no' (but) +то | conjunction and form of `that' +все | all +она | she +так | so, thus +его | him +но | but +да | yes/and +ты | thou +к | towards, by +у | around, chez +же | intensifier particle +вы | you +за | beyond, behind +бы | conditional/subj. particle +по | up to, along +только | only +ее | her +мне | to me +было | it was +вот | here is/are, particle +от | away from +меня | me +еще | still, yet, more +нет | no, there isnt/arent +о | about +из | out of +ему | to him +теперь | now +когда | when +даже | even +ну | so, well +вдруг | suddenly +ли | interrogative particle +если | if +уже | already, but homonym of `narrower' +или | or +ни | neither +быть | to be +был | he was +него | prepositional form of его +до | up to +вас | you accusative +нибудь | indef. suffix preceded by hyphen +опять | again +уж | already, but homonym of `adder' +вам | to you +сказал | he said +ведь | particle `after all' +там | there +потом | then +себя | oneself +ничего | nothing +ей | to her +может | usually with `быть' as `maybe' +они | they +тут | here +где | where +есть | there is/are +надо | got to, must +ней | prepositional form of ей +для | for +мы | we +тебя | thee +их | them, their +чем | than +была | she was +сам | self +чтоб | in order to +без | without +будто | as if +человек | man, person, one +чего | genitive form of `what' +раз | once +тоже | also +себе | to oneself +под | beneath +жизнь | life +будет | will be +ж | short form of intensifer particle `же' +тогда | then +кто | who +этот | this +говорил | was saying +того | genitive form of `that' +потому | for that reason +этого | genitive form of `this' +какой | which +совсем | altogether +ним | prepositional form of `его', `они' +здесь | here +этом | prepositional form of `этот' +один | one +почти | almost +мой | my +тем | instrumental/dative plural of `тот', `то' +чтобы | full form of `in order that' +нее | her (acc.) +кажется | it seems +сейчас | now +были | they were +куда | where to +зачем | why +сказать | to say +всех | all (acc., gen. preposn. plural) +никогда | never +сегодня | today +можно | possible, one can +при | by +наконец | finally +два | two +об | alternative form of `о', about +другой | another +хоть | even +после | after +над | above +больше | more +тот | that one (masc.) +через | across, in +эти | these +нас | us +про | about +всего | in all, only, of all +них | prepositional form of `они' (they) +какая | which, feminine +много | lots +разве | interrogative particle +сказала | she said +три | three +эту | this, acc. fem. sing. +моя | my, feminine +впрочем | moreover, besides +хорошо | good +свою | ones own, acc. fem. sing. +этой | oblique form of `эта', fem. `this' +перед | in front of +иногда | sometimes +лучше | better +чуть | a little +том | preposn. form of `that one' +нельзя | one must not +такой | such a one +им | to them +более | more +всегда | always +конечно | of course +всю | acc. fem. sing of `all' +между | between + + + | b: some paradigms + | + | personal pronouns + | + | я меня мне мной [мною] + | ты тебя тебе тобой [тобою] + | он его ему им [него, нему, ним] + | она ее эи ею [нее, нэи, нею] + | оно его ему им [него, нему, ним] + | + | мы нас нам нами + | вы вас вам вами + | они их им ими [них, ним, ними] + | + | себя себе собой [собою] + | + | demonstrative pronouns: этот (this), тот (that) + | + | этот эта это эти + | этого эты это эти + | этого этой этого этих + | этому этой этому этим + | этим этой этим [этою] этими + | этом этой этом этих + | + | тот та то те + | того ту то те + | того той того тех + | тому той тому тем + | тем той тем [тою] теми + | том той том тех + | + | determinative pronouns + | + | (a) весь (all) + | + | весь вся все все + | всего всю все все + | всего всей всего всех + | всему всей всему всем + | всем всей всем [всею] всеми + | всем всей всем всех + | + | (b) сам (himself etc) + | + | сам сама само сами + | самого саму само самих + | самого самой самого самих + | самому самой самому самим + | самим самой самим [самою] самими + | самом самой самом самих + | + | stems of verbs `to be', `to have', `to do' and modal + | + | быть бы буд быв есть суть + | име + | дел + | мог мож мочь + | уме + | хоч хот + | долж + | можн + | нужн + | нельзя + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_sv.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_sv.txt new file mode 100644 index 000000000..096f87f67 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_sv.txt @@ -0,0 +1,133 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/swedish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Swedish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + | Swedish stop words occasionally exhibit homonym clashes. For example + | så = so, but also seed. These are indicated clearly below. + +och | and +det | it, this/that +att | to (with infinitive) +i | in, at +en | a +jag | I +hon | she +som | who, that +han | he +på | on +den | it, this/that +med | with +var | where, each +sig | him(self) etc +för | for +så | so (also: seed) +till | to +är | is +men | but +ett | a +om | if; around, about +hade | had +de | they, these/those +av | of +icke | not, no +mig | me +du | you +henne | her +då | then, when +sin | his +nu | now +har | have +inte | inte någon = no one +hans | his +honom | him +skulle | 'sake' +hennes | her +där | there +min | my +man | one (pronoun) +ej | nor +vid | at, by, on (also: vast) +kunde | could +något | some etc +från | from, off +ut | out +när | when +efter | after, behind +upp | up +vi | we +dem | them +vara | be +vad | what +över | over +än | than +dig | you +kan | can +sina | his +här | here +ha | have +mot | towards +alla | all +under | under (also: wonder) +någon | some etc +eller | or (else) +allt | all +mycket | much +sedan | since +ju | why +denna | this/that +själv | myself, yourself etc +detta | this/that +åt | to +utan | without +varit | was +hur | how +ingen | no +mitt | my +ni | you +bli | to be, become +blev | from bli +oss | us +din | thy +dessa | these/those +några | some etc +deras | their +blir | from bli +mina | my +samma | (the) same +vilken | who, that +er | you, your +sådan | such a +vår | our +blivit | from bli +dess | its +inom | within +mellan | between +sådant | such a +varför | why +varje | each +vilka | who, that +ditt | thy +vem | who +vilket | who, that +sitta | his +sådana | such a +vart | each +dina | thy +vars | whose +vårt | our +våra | our +ert | your +era | your +vilkas | whose + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_th.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_th.txt new file mode 100644 index 000000000..07f0fabe6 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_th.txt @@ -0,0 +1,119 @@ +# Thai stopwords from: +# "Opinion Detection in Thai Political News Columns +# Based on Subjectivity Analysis" +# Khampol Sukhum, Supot Nitsuwat, and Choochart Haruechaiyasak +ไว้ +ไม่ +ไป +ได้ +ให้ +ใน +โดย +แห่ง +แล้ว +และ +แรก +แบบ +แต่ +เอง +เห็น +เลย +เริ่ม +เรา +เมื่อ +เพื่อ +เพราะ +เป็นการ +เป็น +เปิดเผย +เปิด +เนื่องจาก +เดียวกัน +เดียว +เช่น +เฉพาะ +เคย +เข้า +เขา +อีก +อาจ +อะไร +ออก +อย่าง +อยู่ +อยาก +หาก +หลาย +หลังจาก +หลัง +หรือ +หนึ่ง +ส่วน +ส่ง +สุด +สําหรับ +ว่า +วัน +ลง +ร่วม +ราย +รับ +ระหว่าง +รวม +ยัง +มี +มาก +มา +พร้อม +พบ +ผ่าน +ผล +บาง +น่า +นี้ +นํา +นั้น +นัก +นอกจาก +ทุก +ที่สุด +ที่ +ทําให้ +ทํา +ทาง +ทั้งนี้ +ทั้ง +ถ้า +ถูก +ถึง +ต้อง +ต่างๆ +ต่าง +ต่อ +ตาม +ตั้งแต่ +ตั้ง +ด้าน +ด้วย +ดัง +ซึ่ง +ช่วง +จึง +จาก +จัด +จะ +คือ +ความ +ครั้ง +คง +ขึ้น +ของ +ขอ +ขณะ +ก่อน +ก็ +การ +กับ +กัน +กว่า +กล่าว diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_tr.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_tr.txt new file mode 100644 index 000000000..84d9408d4 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_tr.txt @@ -0,0 +1,212 @@ +# Turkish stopwords from LUCENE-559 +# merged with the list from "Information Retrieval on Turkish Texts" +# (http://www.users.muohio.edu/canf/papers/JASIST2008offPrint.pdf) +acaba +altmış +altı +ama +ancak +arada +aslında +ayrıca +bana +bazı +belki +ben +benden +beni +benim +beri +beş +bile +bin +bir +birçok +biri +birkaç +birkez +birşey +birşeyi +biz +bize +bizden +bizi +bizim +böyle +böylece +bu +buna +bunda +bundan +bunlar +bunları +bunların +bunu +bunun +burada +çok +çünkü +da +daha +dahi +de +defa +değil +diğer +diye +doksan +dokuz +dolayı +dolayısıyla +dört +edecek +eden +ederek +edilecek +ediliyor +edilmesi +ediyor +eğer +elli +en +etmesi +etti +ettiği +ettiğini +gibi +göre +halen +hangi +hatta +hem +henüz +hep +hepsi +her +herhangi +herkesin +hiç +hiçbir +için +iki +ile +ilgili +ise +işte +itibaren +itibariyle +kadar +karşın +katrilyon +kendi +kendilerine +kendini +kendisi +kendisine +kendisini +kez +ki +kim +kimden +kime +kimi +kimse +kırk +milyar +milyon +mu +mü +mı +nasıl +ne +neden +nedenle +nerde +nerede +nereye +niye +niçin +o +olan +olarak +oldu +olduğu +olduğunu +olduklarını +olmadı +olmadığı +olmak +olması +olmayan +olmaz +olsa +olsun +olup +olur +olursa +oluyor +on +ona +ondan +onlar +onlardan +onları +onların +onu +onun +otuz +oysa +öyle +pek +rağmen +sadece +sanki +sekiz +seksen +sen +senden +seni +senin +siz +sizden +sizi +sizin +şey +şeyden +şeyi +şeyler +şöyle +şu +şuna +şunda +şundan +şunları +şunu +tarafından +trilyon +tüm +üç +üzere +var +vardı +ve +veya +ya +yani +yapacak +yapılan +yapılması +yapıyor +yapmak +yaptı +yaptığı +yaptığını +yaptıkları +yedi +yerine +yetmiş +yine +yirmi +yoksa +yüz +zaten diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/userdict_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/userdict_ja.txt new file mode 100644 index 000000000..6f0368e4d --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/userdict_ja.txt @@ -0,0 +1,29 @@ +# +# This is a sample user dictionary for Kuromoji (JapaneseTokenizer) +# +# Add entries to this file in order to override the statistical model in terms +# of segmentation, readings and part-of-speech tags. Notice that entries do +# not have weights since they are always used when found. This is by-design +# in order to maximize ease-of-use. +# +# Entries are defined using the following CSV format: +# , ... , ... , +# +# Notice that a single half-width space separates tokens and readings, and +# that the number tokens and readings must match exactly. +# +# Also notice that multiple entries with the same is undefined. +# +# Whitespace only lines are ignored. Comments are not allowed on entry lines. +# + +# Custom segmentation for kanji compounds +日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞 +関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,カスタム名詞 + +# Custom segmentation for compound katakana +トートバッグ,トート バッグ,トート バッグ,かずカナ名詞 +ショルダーバッグ,ショルダー バッグ,ショルダー バッグ,かずカナ名詞 + +# Custom reading for former sumo wrestler +朝青龍,朝青龍,アサショウリュウ,カスタム人名 diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/protwords.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/protwords.txt new file mode 100644 index 000000000..1dfc0abec --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/schema-rerank.xml b/search-services/alfresco-search/src/test/resources/test-files/master/conf/schema-rerank.xml new file mode 100644 index 000000000..6cd4159df --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/schema-rerank.xml @@ -0,0 +1,410 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/schema.xml b/search-services/alfresco-search/src/test/resources/test-files/master/conf/schema.xml new file mode 100644 index 000000000..024c6ebb2 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/schema.xml @@ -0,0 +1,766 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.snippet.randomindexconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.snippet.randomindexconfig.xml new file mode 100644 index 000000000..7514aa478 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.snippet.randomindexconfig.xml @@ -0,0 +1,47 @@ + + + + + + + + + ${useCompoundFile:false} + + ${solr.tests.maxBufferedDocs} + ${solr.tests.maxIndexingThreads} + ${solr.tests.ramBufferSizeMB} + + + + 1000 + 10000 + + + ${solr.tests.lockType:single} + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml new file mode 100644 index 000000000..a65aefcfe --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml @@ -0,0 +1,570 @@ + + + + + + + + + + + + ${solr.data.dir:} + + + + 1000000 + 2000000 + 3000000 + 4000000 + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + true + + + + + + 10 + + + + + + + + + + + + + + + + + + + + + + commit + schema.xml + /Users/agazzarini/workspaces/alfresco/spike-custom-replication-handler/src/test/resources/contentstore + + + + + + + + + true + + + + + true + + + + + + dismax + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + + + fingerprint + + + + + + afts + false + false + 5 + 2 + 5 + true + true + 5 + 3 + mltext@m___t@{http://www.alfresco.org/model/content/1.0}title + id + content@s___t@{http://www.alfresco.org/model/content/1.0}content + true + false + + + setLocale + rewriteFacetParameters + query + facet + facet_module + mlt + highlight + stats + debug + clearLocale + rewriteFacetCounts + spellcheck + spellcheckbackcompat + setProcessedDenies + + + + + + + + explicit + 10 + suggest + + + + + setLocale + query + facet + mlt + highlight + stats + debug + clearLocale + + + + + + + + cmis + + + query + facet + mlt + highlight + stats + debug + + + + + + + + + termsComp + + + + + + + + + + + + + + + tvComponent + + + + + + + + + + + + 100 + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + max-age=30, public + + + + + + + explicit + true + + + + + solr + solrconfig.xml schema.xml admin-extra.html + + + + + + + + + + + + + + + + + + conf/mime_types.csv + + + + 1 + 10 + + + + + + + + text_shingle + + + + + + default + suggest + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + wordbreak + suggest + solr.WordBreakSolrSpellChecker + true + true + 10 + 5 + + + + + + + + + + + + + + + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrcore.properties b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrcore.properties new file mode 100644 index 000000000..aa07843d0 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrcore.properties @@ -0,0 +1,154 @@ +# +# solrcore.properties - used in solrconfig.xml +# + +enable.alfresco.tracking=true + +# +#These are replaced by the admin handler +# +#data.dir.root=DATA_DIR +#data.dir.store=workspace/SpacesStore +#alfresco.stores=workspace://SpacesStore + +# +# Properties loaded during alfresco tracking +# + +alfresco.host=localhost +alfresco.port=8080 +alfresco.port.ssl=8443 +alfresco.baseUrl=/alfresco +alfresco.cron=0/2 * * * * ? * + +#alfresco.index.transformContent=false +#alfresco.ignore.datatype.1=d:content +alfresco.lag=1000 +alfresco.hole.retention=3600000 +# alfresco.hole.check.after is not used yet +# It will reduce the hole checking load +alfresco.hole.check.after=300000 +alfresco.batch.count=1000 + +# encryption + +# none, https +alfresco.secureComms=none + +# ssl +alfresco.encryption.ssl.keystore.type=JCEKS +alfresco.encryption.ssl.keystore.provider= +alfresco.encryption.ssl.keystore.location=ssl.repo.client.keystore +alfresco.encryption.ssl.keystore.passwordFileLocation=ssl-keystore-passwords.properties +alfresco.encryption.ssl.truststore.type=JCEKS +alfresco.encryption.ssl.truststore.provider= +alfresco.encryption.ssl.truststore.location=ssl.repo.client.truststore +alfresco.encryption.ssl.truststore.passwordFileLocation=ssl-truststore-passwords.properties + +# Tracking + +alfresco.corePoolSize=1 +alfresco.maximumPoolSize=-1 +alfresco.keepAliveTime=120 +alfresco.threadPriority=5 +alfresco.threadDaemon=true +alfresco.workQueueSize=-1 + +# HTTP Client + +alfresco.maxTotalConnections=200 +alfresco.maxHostConnections=200 +alfresco.socketTimeout=360000 + +# SOLR caching + +solr.filterCache.size=256 +solr.filterCache.initialSize=128 +solr.queryResultCache.size=1024 +solr.queryResultCache.initialSize=1024 +solr.documentCache.size=1024 +solr.documentCache.initialSize=1024 +solr.queryResultMaxDocsCached=2048 + +solr.authorityCache.size=128 +solr.authorityCache.initialSize=64 +solr.pathCache.size=256 +solr.pathCache.initialSize=128 + +solr.ownerCache.size=128 +solr.ownerCache.initialSize=64 + +solr.readerCache.size=128 +solr.readerCache.initialSize=64 + +solr.deniedCache.size=128 +solr.deniedCache.initialSize=64 + +# SOLR + +solr.maxBooleanClauses=10000 + +# Batch fetch + +alfresco.transactionDocsBatchSize=100 +alfresco.nodeBatchSize=10 +alfresco.changeSetAclsBatchSize=100 +alfresco.aclBatchSize=10 +alfresco.contentReadBatchSize=4000 +alfresco.contentUpdateBatchSize=1000 + +# Warming + +solr.filterCache.autowarmCount=32 +solr.authorityCache.autowarmCount=4 +solr.pathCache.autowarmCount=32 +solr.deniedCache.autowarmCount=0 +solr.readerCache.autowarmCount=0 +solr.ownerCache.autowarmCount=0 +solr.queryResultCache.autowarmCount=4 +solr.documentCache.autowarmCount=512 + +solr.queryResultWindowSize=512 + + +# +# TODO +# +# cross language support +# locale expansion +# logging check report .... +# +# + +alfresco.commitInterval=1000 +alfresco.newSearcherInterval=2000 + +alfresco.doPermissionChecks=true + + +# +# Metadata pulling control +# +alfresco.metadata.skipDescendantDocsForSpecificTypes=false +alfresco.metadata.ignore.datatype.0=cm:person +alfresco.metadata.ignore.datatype.1=app:configurations +alfresco.metadata.skipDescendantDocsForSpecificAspects=false +#alfresco.metadata.ignore.aspect.0= + + +# +# Suggestions +# +solr.suggester.enabled=false +# -1 to disable suggester build throttling +solr.suggester.minSecsBetweenBuilds=3600 + +# +# Limit the maximum text size of transformed content sent to the index - in bytes +# +alfresco.contentStreamLimit=10000000 + +#Sharding default values +shard.instance=1 +shard.count=0 +shard.method=DB_ID diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/spellings.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/spellings.txt new file mode 100644 index 000000000..d7ede6f56 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/spellings.txt @@ -0,0 +1,2 @@ +pizza +history \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/stopwords.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/stopwords.txt new file mode 100644 index 000000000..6f46b1bf4 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/stopwords.txt @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/synonyms.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/synonyms.txt new file mode 100644 index 000000000..74cea68ad --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/synonyms.txt @@ -0,0 +1,35 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaa => aaaa +bbb => bbbb1 bbbb2 +ccc => cccc1,cccc2 +a\=>a => b\=>b +a\,a => b\,b +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + +# Test synonyms +quick,fast,rapid,speedy +brown fox jumped,leaping reynard,springer +lazy,bone idle \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/README.md b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/README.md new file mode 100644 index 000000000..47c6e62d2 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/README.md @@ -0,0 +1,14 @@ +The folder contains several Solr configuration files that are used within the test suite. +Note that the folder, although it has the structure that Solr expects (/conf), it is not a complete "core" +folder, because + +- some files are missing (e.g. schema.xml) +- some files are used only in specific tests (e.g. solrconfig-rerank.xml, schema-rerank.xml) + +During the build process, Maven creates a complete core definition under the test build output folder by merging the +configuration of the "Rerank" template (src/main/resources/templates/rerank) together with the content of this folder. + +Note that this folder is copied **after** the template, so duplicates will be overwritten. For example, both folders +have a *solrcore.properties* and a *solrconfig.xml*: the test execution will use are those in this folder (because they +will overwrite the same files in the rerank template). + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.html b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.html new file mode 100644 index 000000000..d8f22b44e --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.html @@ -0,0 +1,156 @@ + + + + + +
Update the Summary and FTS Status reports
+ +
+
+

Alfresco Core - Summary Report

+
+
+
+
+
+
+
+
+
+
+

Alfresco Core - FTS Status Report

+
+
+
+
+
+
+
+
+
+
+ +
+ + + + +

Other Links:

+
+
Note: the following links in a new window
+ + + +
diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.menu-bottom.html b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.menu-bottom.html new file mode 100644 index 000000000..e69de29bb diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.menu-top.html b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.menu-top.html new file mode 100644 index 000000000..e69de29bb diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/elevate.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/elevate.xml new file mode 100644 index 000000000..ed2bc4c3a --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/elevate.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ca.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ca.txt new file mode 100644 index 000000000..307a85f91 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ca.txt @@ -0,0 +1,8 @@ +# Set of Catalan contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +d +l +m +n +s +t diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_fr.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_fr.txt new file mode 100644 index 000000000..f1bba51b2 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_fr.txt @@ -0,0 +1,15 @@ +# Set of French contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +l +m +t +qu +n +s +j +d +c +jusqu +quoiqu +lorsqu +puisqu diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ga.txt new file mode 100644 index 000000000..9ebe7fa34 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ga.txt @@ -0,0 +1,5 @@ +# Set of Irish contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +d +m +b diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_it.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_it.txt new file mode 100644 index 000000000..cac040953 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_it.txt @@ -0,0 +1,23 @@ +# Set of Italian contractions for ElisionFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +c +l +all +dall +dell +nell +sull +coll +pell +gl +agl +dagl +degl +negl +sugl +un +m +t +s +v +d diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/hyphenations_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/hyphenations_ga.txt new file mode 100644 index 000000000..4d2642cc5 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/hyphenations_ga.txt @@ -0,0 +1,5 @@ +# Set of Irish hyphenations for StopFilter +# TODO: load this as a resource from the analyzer and sync it in build.xml +h +n +t diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stemdict_nl.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stemdict_nl.txt new file mode 100644 index 000000000..441072971 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stemdict_nl.txt @@ -0,0 +1,6 @@ +# Set of overrides for the dutch stemmer +# TODO: load this as a resource from the analyzer and sync it in build.xml +fiets fiets +bromfiets bromfiets +ei eier +kind kinder diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stoptags_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stoptags_ja.txt new file mode 100644 index 000000000..71b750845 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stoptags_ja.txt @@ -0,0 +1,420 @@ +# +# This file defines a Japanese stoptag set for JapanesePartOfSpeechStopFilter. +# +# Any token with a part-of-speech tag that exactly matches those defined in this +# file are removed from the token stream. +# +# Set your own stoptags by uncommenting the lines below. Note that comments are +# not allowed on the same line as a stoptag. See LUCENE-3745 for frequency lists, +# etc. that can be useful for building you own stoptag set. +# +# The entire possible tagset is provided below for convenience. +# +##### +# noun: unclassified nouns +#名詞 +# +# noun-common: Common nouns or nouns where the sub-classification is undefined +#名詞-一般 +# +# noun-proper: Proper nouns where the sub-classification is undefined +#名詞-固有名詞 +# +# noun-proper-misc: miscellaneous proper nouns +#名詞-固有名詞-一般 +# +# noun-proper-person: Personal names where the sub-classification is undefined +#名詞-固有名詞-人名 +# +# noun-proper-person-misc: names that cannot be divided into surname and +# given name; foreign names; names where the surname or given name is unknown. +# e.g. お市の方 +#名詞-固有名詞-人名-一般 +# +# noun-proper-person-surname: Mainly Japanese surnames. +# e.g. 山田 +#名詞-固有名詞-人名-姓 +# +# noun-proper-person-given_name: Mainly Japanese given names. +# e.g. 太郎 +#名詞-固有名詞-人名-名 +# +# noun-proper-organization: Names representing organizations. +# e.g. 通産省, NHK +#名詞-固有名詞-組織 +# +# noun-proper-place: Place names where the sub-classification is undefined +#名詞-固有名詞-地域 +# +# noun-proper-place-misc: Place names excluding countries. +# e.g. アジア, バルセロナ, 京都 +#名詞-固有名詞-地域-一般 +# +# noun-proper-place-country: Country names. +# e.g. 日本, オーストラリア +#名詞-固有名詞-地域-国 +# +# noun-pronoun: Pronouns where the sub-classification is undefined +#名詞-代名詞 +# +# noun-pronoun-misc: miscellaneous pronouns: +# e.g. それ, ここ, あいつ, あなた, あちこち, いくつ, どこか, なに, みなさん, みんな, わたくし, われわれ +#名詞-代名詞-一般 +# +# noun-pronoun-contraction: Spoken language contraction made by combining a +# pronoun and the particle 'wa'. +# e.g. ありゃ, こりゃ, こりゃあ, そりゃ, そりゃあ +#名詞-代名詞-縮約 +# +# noun-adverbial: Temporal nouns such as names of days or months that behave +# like adverbs. Nouns that represent amount or ratios and can be used adverbially, +# e.g. 金曜, 一月, 午後, 少量 +#名詞-副詞可能 +# +# noun-verbal: Nouns that take arguments with case and can appear followed by +# 'suru' and related verbs (する, できる, なさる, くださる) +# e.g. インプット, 愛着, 悪化, 悪戦苦闘, 一安心, 下取り +#名詞-サ変接続 +# +# noun-adjective-base: The base form of adjectives, words that appear before な ("na") +# e.g. 健康, 安易, 駄目, だめ +#名詞-形容動詞語幹 +# +# noun-numeric: Arabic numbers, Chinese numerals, and counters like 何 (回), 数. +# e.g. 0, 1, 2, 何, 数, 幾 +#名詞-数 +# +# noun-affix: noun affixes where the sub-classification is undefined +#名詞-非自立 +# +# noun-affix-misc: Of adnominalizers, the case-marker の ("no"), and words that +# attach to the base form of inflectional words, words that cannot be classified +# into any of the other categories below. This category includes indefinite nouns. +# e.g. あかつき, 暁, かい, 甲斐, 気, きらい, 嫌い, くせ, 癖, こと, 事, ごと, 毎, しだい, 次第, +# 順, せい, 所為, ついで, 序で, つもり, 積もり, 点, どころ, の, はず, 筈, はずみ, 弾み, +# 拍子, ふう, ふり, 振り, ほう, 方, 旨, もの, 物, 者, ゆえ, 故, ゆえん, 所以, わけ, 訳, +# わり, 割り, 割, ん-口語/, もん-口語/ +#名詞-非自立-一般 +# +# noun-affix-adverbial: noun affixes that that can behave as adverbs. +# e.g. あいだ, 間, あげく, 挙げ句, あと, 後, 余り, 以外, 以降, 以後, 以上, 以前, 一方, うえ, +# 上, うち, 内, おり, 折り, かぎり, 限り, きり, っきり, 結果, ころ, 頃, さい, 際, 最中, さなか, +# 最中, じたい, 自体, たび, 度, ため, 為, つど, 都度, とおり, 通り, とき, 時, ところ, 所, +# とたん, 途端, なか, 中, のち, 後, ばあい, 場合, 日, ぶん, 分, ほか, 他, まえ, 前, まま, +# 儘, 侭, みぎり, 矢先 +#名詞-非自立-副詞可能 +# +# noun-affix-aux: noun affixes treated as 助動詞 ("auxiliary verb") in school grammars +# with the stem よう(だ) ("you(da)"). +# e.g. よう, やう, 様 (よう) +#名詞-非自立-助動詞語幹 +# +# noun-affix-adjective-base: noun affixes that can connect to the indeclinable +# connection form な (aux "da"). +# e.g. みたい, ふう +#名詞-非自立-形容動詞語幹 +# +# noun-special: special nouns where the sub-classification is undefined. +#名詞-特殊 +# +# noun-special-aux: The そうだ ("souda") stem form that is used for reporting news, is +# treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the base +# form of inflectional words. +# e.g. そう +#名詞-特殊-助動詞語幹 +# +# noun-suffix: noun suffixes where the sub-classification is undefined. +#名詞-接尾 +# +# noun-suffix-misc: Of the nouns or stem forms of other parts of speech that connect +# to ガル or タイ and can combine into compound nouns, words that cannot be classified into +# any of the other categories below. In general, this category is more inclusive than +# 接尾語 ("suffix") and is usually the last element in a compound noun. +# e.g. おき, かた, 方, 甲斐 (がい), がかり, ぎみ, 気味, ぐるみ, (～した) さ, 次第, 済 (ず) み, +# よう, (でき)っこ, 感, 観, 性, 学, 類, 面, 用 +#名詞-接尾-一般 +# +# noun-suffix-person: Suffixes that form nouns and attach to person names more often +# than other nouns. +# e.g. 君, 様, 著 +#名詞-接尾-人名 +# +# noun-suffix-place: Suffixes that form nouns and attach to place names more often +# than other nouns. +# e.g. 町, 市, 県 +#名詞-接尾-地域 +# +# noun-suffix-verbal: Of the suffixes that attach to nouns and form nouns, those that +# can appear before スル ("suru"). +# e.g. 化, 視, 分け, 入り, 落ち, 買い +#名詞-接尾-サ変接続 +# +# noun-suffix-aux: The stem form of そうだ (様態) that is used to indicate conditions, +# is treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the +# conjunctive form of inflectional words. +# e.g. そう +#名詞-接尾-助動詞語幹 +# +# noun-suffix-adjective-base: Suffixes that attach to other nouns or the conjunctive +# form of inflectional words and appear before the copula だ ("da"). +# e.g. 的, げ, がち +#名詞-接尾-形容動詞語幹 +# +# noun-suffix-adverbial: Suffixes that attach to other nouns and can behave as adverbs. +# e.g. 後 (ご), 以後, 以降, 以前, 前後, 中, 末, 上, 時 (じ) +#名詞-接尾-副詞可能 +# +# noun-suffix-classifier: Suffixes that attach to numbers and form nouns. This category +# is more inclusive than 助数詞 ("classifier") and includes common nouns that attach +# to numbers. +# e.g. 個, つ, 本, 冊, パーセント, cm, kg, カ月, か国, 区画, 時間, 時半 +#名詞-接尾-助数詞 +# +# noun-suffix-special: Special suffixes that mainly attach to inflecting words. +# e.g. (楽し) さ, (考え) 方 +#名詞-接尾-特殊 +# +# noun-suffix-conjunctive: Nouns that behave like conjunctions and join two words +# together. +# e.g. (日本) 対 (アメリカ), 対 (アメリカ), (3) 対 (5), (女優) 兼 (主婦) +#名詞-接続詞的 +# +# noun-verbal_aux: Nouns that attach to the conjunctive particle て ("te") and are +# semantically verb-like. +# e.g. ごらん, ご覧, 御覧, 頂戴 +#名詞-動詞非自立的 +# +# noun-quotation: text that cannot be segmented into words, proverbs, Chinese poetry, +# dialects, English, etc. Currently, the only entry for 名詞 引用文字列 ("noun quotation") +# is いわく ("iwaku"). +#名詞-引用文字列 +# +# noun-nai_adjective: Words that appear before the auxiliary verb ない ("nai") and +# behave like an adjective. +# e.g. 申し訳, 仕方, とんでも, 違い +#名詞-ナイ形容詞語幹 +# +##### +# prefix: unclassified prefixes +#接頭詞 +# +# prefix-nominal: Prefixes that attach to nouns (including adjective stem forms) +# excluding numerical expressions. +# e.g. お (水), 某 (氏), 同 (社), 故 (～氏), 高 (品質), お (見事), ご (立派) +#接頭詞-名詞接続 +# +# prefix-verbal: Prefixes that attach to the imperative form of a verb or a verb +# in conjunctive form followed by なる/なさる/くださる. +# e.g. お (読みなさい), お (座り) +#接頭詞-動詞接続 +# +# prefix-adjectival: Prefixes that attach to adjectives. +# e.g. お (寒いですねえ), バカ (でかい) +#接頭詞-形容詞接続 +# +# prefix-numerical: Prefixes that attach to numerical expressions. +# e.g. 約, およそ, 毎時 +#接頭詞-数接続 +# +##### +# verb: unclassified verbs +#動詞 +# +# verb-main: +#動詞-自立 +# +# verb-auxiliary: +#動詞-非自立 +# +# verb-suffix: +#動詞-接尾 +# +##### +# adjective: unclassified adjectives +#形容詞 +# +# adjective-main: +#形容詞-自立 +# +# adjective-auxiliary: +#形容詞-非自立 +# +# adjective-suffix: +#形容詞-接尾 +# +##### +# adverb: unclassified adverbs +#副詞 +# +# adverb-misc: Words that can be segmented into one unit and where adnominal +# modification is not possible. +# e.g. あいかわらず, 多分 +#副詞-一般 +# +# adverb-particle_conjunction: Adverbs that can be followed by の, は, に, +# な, する, だ, etc. +# e.g. こんなに, そんなに, あんなに, なにか, なんでも +#副詞-助詞類接続 +# +##### +# adnominal: Words that only have noun-modifying forms. +# e.g. この, その, あの, どの, いわゆる, なんらかの, 何らかの, いろんな, こういう, そういう, ああいう, +# どういう, こんな, そんな, あんな, どんな, 大きな, 小さな, おかしな, ほんの, たいした, +# 「(, も) さる (ことながら)」, 微々たる, 堂々たる, 単なる, いかなる, 我が」「同じ, 亡き +#連体詞 +# +##### +# conjunction: Conjunctions that can occur independently. +# e.g. が, けれども, そして, じゃあ, それどころか +接続詞 +# +##### +# particle: unclassified particles. +助詞 +# +# particle-case: case particles where the subclassification is undefined. +助詞-格助詞 +# +# particle-case-misc: Case particles. +# e.g. から, が, で, と, に, へ, より, を, の, にて +助詞-格助詞-一般 +# +# particle-case-quote: the "to" that appears after nouns, a person’s speech, +# quotation marks, expressions of decisions from a meeting, reasons, judgements, +# conjectures, etc. +# e.g. ( だ) と (述べた.), ( である) と (して執行猶予...) +助詞-格助詞-引用 +# +# particle-case-compound: Compounds of particles and verbs that mainly behave +# like case particles. +# e.g. という, といった, とかいう, として, とともに, と共に, でもって, にあたって, に当たって, に当って, +# にあたり, に当たり, に当り, に当たる, にあたる, において, に於いて,に於て, における, に於ける, +# にかけ, にかけて, にかんし, に関し, にかんして, に関して, にかんする, に関する, に際し, +# に際して, にしたがい, に従い, に従う, にしたがって, に従って, にたいし, に対し, にたいして, +# に対して, にたいする, に対する, について, につき, につけ, につけて, につれ, につれて, にとって, +# にとり, にまつわる, によって, に依って, に因って, により, に依り, に因り, による, に依る, に因る, +# にわたって, にわたる, をもって, を以って, を通じ, を通じて, を通して, をめぐって, をめぐり, をめぐる, +# って-口語/, ちゅう-関西弁「という」/, (何) ていう (人)-口語/, っていう-口語/, といふ, とかいふ +助詞-格助詞-連語 +# +# particle-conjunctive: +# e.g. から, からには, が, けれど, けれども, けど, し, つつ, て, で, と, ところが, どころか, とも, ども, +# ながら, なり, ので, のに, ば, ものの, や ( した), やいなや, (ころん) じゃ(いけない)-口語/, +# (行っ) ちゃ(いけない)-口語/, (言っ) たって (しかたがない)-口語/, (それがなく)ったって (平気)-口語/ +助詞-接続助詞 +# +# particle-dependency: +# e.g. こそ, さえ, しか, すら, は, も, ぞ +助詞-係助詞 +# +# particle-adverbial: +# e.g. がてら, かも, くらい, 位, ぐらい, しも, (学校) じゃ(これが流行っている)-口語/, +# (それ)じゃあ (よくない)-口語/, ずつ, (私) なぞ, など, (私) なり (に), (先生) なんか (大嫌い)-口語/, +# (私) なんぞ, (先生) なんて (大嫌い)-口語/, のみ, だけ, (私) だって-口語/, だに, +# (彼)ったら-口語/, (お茶) でも (いかが), 等 (とう), (今後) とも, ばかり, ばっか-口語/, ばっかり-口語/, +# ほど, 程, まで, 迄, (誰) も (が)([助詞-格助詞] および [助詞-係助詞] の前に位置する「も」) +助詞-副助詞 +# +# particle-interjective: particles with interjective grammatical roles. +# e.g. (松島) や +助詞-間投助詞 +# +# particle-coordinate: +# e.g. と, たり, だの, だり, とか, なり, や, やら +助詞-並立助詞 +# +# particle-final: +# e.g. かい, かしら, さ, ぜ, (だ)っけ-口語/, (とまってる) で-方言/, な, ナ, なあ-口語/, ぞ, ね, ネ, +# ねぇ-口語/, ねえ-口語/, ねん-方言/, の, のう-口語/, や, よ, ヨ, よぉ-口語/, わ, わい-口語/ +助詞-終助詞 +# +# particle-adverbial/conjunctive/final: The particle "ka" when unknown whether it is +# adverbial, conjunctive, or sentence final. For example: +# (a) 「A か B か」. Ex:「(国内で運用する) か,(海外で運用する) か (.)」 +# (b) Inside an adverb phrase. Ex:「(幸いという) か (, 死者はいなかった.)」 +# 「(祈りが届いたせい) か (, 試験に合格した.)」 +# (c) 「かのように」. Ex:「(何もなかった) か (のように振る舞った.)」 +# e.g. か +助詞-副助詞／並立助詞／終助詞 +# +# particle-adnominalizer: The "no" that attaches to nouns and modifies +# non-inflectional words. +助詞-連体化 +# +# particle-adnominalizer: The "ni" and "to" that appear following nouns and adverbs +# that are giongo, giseigo, or gitaigo. +# e.g. に, と +助詞-副詞化 +# +# particle-special: A particle that does not fit into one of the above classifications. +# This includes particles that are used in Tanka, Haiku, and other poetry. +# e.g. かな, けむ, ( しただろう) に, (あんた) にゃ(わからん), (俺) ん (家) +助詞-特殊 +# +##### +# auxiliary-verb: +助動詞 +# +##### +# interjection: Greetings and other exclamations. +# e.g. おはよう, おはようございます, こんにちは, こんばんは, ありがとう, どうもありがとう, ありがとうございます, +# いただきます, ごちそうさま, さよなら, さようなら, はい, いいえ, ごめん, ごめんなさい +#感動詞 +# +##### +# symbol: unclassified Symbols. +記号 +# +# symbol-misc: A general symbol not in one of the categories below. +# e.g. [○◎@$〒→+] +記号-一般 +# +# symbol-comma: Commas +# e.g. [,、] +記号-読点 +# +# symbol-period: Periods and full stops. +# e.g. [.．。] +記号-句点 +# +# symbol-space: Full-width whitespace. +記号-空白 +# +# symbol-open_bracket: +# e.g. [({‘“『【] +記号-括弧開 +# +# symbol-close_bracket: +# e.g. [)}’”』」】] +記号-括弧閉 +# +# symbol-alphabetic: +#記号-アルファベット +# +##### +# other: unclassified other +#その他 +# +# other-interjection: Words that are hard to classify as noun-suffixes or +# sentence-final particles. +# e.g. (だ)ァ +その他-間投 +# +##### +# filler: Aizuchi that occurs during a conversation or sounds inserted as filler. +# e.g. あの, うんと, えと +フィラー +# +##### +# non-verbal: non-verbal sound. +非言語音 +# +##### +# fragment: +#語断片 +# +##### +# unknown: unknown part of speech. +#未知語 +# +##### End of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ar.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ar.txt new file mode 100644 index 000000000..046829db6 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ar.txt @@ -0,0 +1,125 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +# Cleaned on October 11, 2009 (not normalized, so use before normalization) +# This means that when modifying this list, you might need to add some +# redundant entries, for example containing forms with both أ and ا +من +ومن +منها +منه +في +وفي +فيها +فيه +و +ف +ثم +او +أو +ب +بها +به +ا +أ +اى +اي +أي +أى +لا +ولا +الا +ألا +إلا +لكن +ما +وما +كما +فما +عن +مع +اذا +إذا +ان +أن +إن +انها +أنها +إنها +انه +أنه +إنه +بان +بأن +فان +فأن +وان +وأن +وإن +التى +التي +الذى +الذي +الذين +الى +الي +إلى +إلي +على +عليها +عليه +اما +أما +إما +ايضا +أيضا +كل +وكل +لم +ولم +لن +ولن +هى +هي +هو +وهى +وهي +وهو +فهى +فهي +فهو +انت +أنت +لك +لها +له +هذه +هذا +تلك +ذلك +هناك +كانت +كان +يكون +تكون +وكانت +وكان +غير +بعض +قد +نحو +بين +بينما +منذ +ضمن +حيث +الان +الآن +خلال +بعد +قبل +حتى +عند +عندما +لدى +جميع diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_bg.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_bg.txt new file mode 100644 index 000000000..1ae4ba2ae --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_bg.txt @@ -0,0 +1,193 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +а +аз +ако +ала +бе +без +беше +би +бил +била +били +било +близо +бъдат +бъде +бяха +в +вас +ваш +ваша +вероятно +вече +взема +ви +вие +винаги +все +всеки +всички +всичко +всяка +във +въпреки +върху +г +ги +главно +го +д +да +дали +до +докато +докога +дори +досега +доста +е +едва +един +ето +за +зад +заедно +заради +засега +затова +защо +защото +и +из +или +им +има +имат +иска +й +каза +как +каква +какво +както +какъв +като +кога +когато +което +които +кой +който +колко +която +къде +където +към +ли +м +ме +между +мен +ми +мнозина +мога +могат +може +моля +момента +му +н +на +над +назад +най +направи +напред +например +нас +не +него +нея +ни +ние +никой +нито +но +някои +някой +няма +обаче +около +освен +особено +от +отгоре +отново +още +пак +по +повече +повечето +под +поне +поради +после +почти +прави +пред +преди +през +при +пък +първо +с +са +само +се +сега +си +скоро +след +сме +според +сред +срещу +сте +съм +със +също +т +тази +така +такива +такъв +там +твой +те +тези +ти +тн +то +това +тогава +този +той +толкова +точно +трябва +тук +тъй +тя +тях +у +харесва +ч +че +често +чрез +ще +щом +я diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ca.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ca.txt new file mode 100644 index 000000000..3da65deaf --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ca.txt @@ -0,0 +1,220 @@ +# Catalan stopwords from http://github.com/vcl/cue.language (Apache 2 Licensed) +a +abans +ací +ah +així +això +al +als +aleshores +algun +alguna +algunes +alguns +alhora +allà +allí +allò +altra +altre +altres +amb +ambdós +ambdues +apa +aquell +aquella +aquelles +aquells +aquest +aquesta +aquestes +aquests +aquí +baix +cada +cadascú +cadascuna +cadascunes +cadascuns +com +contra +d'un +d'una +d'unes +d'uns +dalt +de +del +dels +des +després +dins +dintre +donat +doncs +durant +e +eh +el +els +em +en +encara +ens +entre +érem +eren +éreu +es +és +esta +està +estàvem +estaven +estàveu +esteu +et +etc +ets +fins +fora +gairebé +ha +han +has +havia +he +hem +heu +hi +ho +i +igual +iguals +ja +l'hi +la +les +li +li'n +llavors +m'he +ma +mal +malgrat +mateix +mateixa +mateixes +mateixos +me +mentre +més +meu +meus +meva +meves +molt +molta +moltes +molts +mon +mons +n'he +n'hi +ne +ni +no +nogensmenys +només +nosaltres +nostra +nostre +nostres +o +oh +oi +on +pas +pel +pels +per +però +perquè +poc +poca +pocs +poques +potser +propi +qual +quals +quan +quant +que +què +quelcom +qui +quin +quina +quines +quins +s'ha +s'han +sa +semblant +semblants +ses +seu +seus +seva +seva +seves +si +sobre +sobretot +sóc +solament +sols +son +són +sons +sota +sou +t'ha +t'han +t'he +ta +tal +també +tampoc +tan +tant +tanta +tantes +teu +teus +teva +teves +ton +tons +tot +tota +totes +tots +un +una +unes +uns +us +va +vaig +vam +van +vas +veu +vosaltres +vostra +vostre +vostres diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ckb.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ckb.txt new file mode 100644 index 000000000..87abf118f --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ckb.txt @@ -0,0 +1,136 @@ +# set of kurdish stopwords +# note these have been normalized with our scheme (e represented with U+06D5, etc) +# constructed from: +# * Fig 5 of "Building A Test Collection For Sorani Kurdish" (Esmaili et al) +# * "Sorani Kurdish: A Reference Grammar with selected readings" (Thackston) +# * Corpus-based analysis of 77M word Sorani collection: wikipedia, news, blogs, etc + +# and +و +# which +کە +# of +ی +# made/did +کرد +# that/which +ئەوەی +# on/head +سەر +# two +دوو +# also +هەروەها +# from/that +لەو +# makes/does +دەکات +# some +چەند +# every +هەر + +# demonstratives +# that +ئەو +# this +ئەم + +# personal pronouns +# I +من +# we +ئێمە +# you +تۆ +# you +ئێوە +# he/she/it +ئەو +# they +ئەوان + +# prepositions +# to/with/by +بە +پێ +# without +بەبێ +# along with/while/during +بەدەم +# in the opinion of +بەلای +# according to +بەپێی +# before +بەرلە +# in the direction of +بەرەوی +# in front of/toward +بەرەوە +# before/in the face of +بەردەم +# without +بێ +# except for +بێجگە +# for +بۆ +# on/in +دە +تێ +# with +دەگەڵ +# after +دوای +# except for/aside from +جگە +# in/from +لە +لێ +# in front of/before/because of +لەبەر +# between/among +لەبەینی +# concerning/about +لەبابەت +# concerning +لەبارەی +# instead of +لەباتی +# beside +لەبن +# instead of +لەبرێتی +# behind +لەدەم +# with/together with +لەگەڵ +# by +لەلایەن +# within +لەناو +# between/among +لەنێو +# for the sake of +لەپێناوی +# with respect to +لەرەوی +# by means of/for +لەرێ +# for the sake of +لەرێگا +# on/on top of/according to +لەسەر +# under +لەژێر +# between/among +ناو +# between/among +نێوان +# after +پاش +# before +پێش +# like +وەک diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_cz.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_cz.txt new file mode 100644 index 000000000..53c6097da --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_cz.txt @@ -0,0 +1,172 @@ +a +s +k +o +i +u +v +z +dnes +cz +tímto +budeš +budem +byli +jseš +můj +svým +ta +tomto +tohle +tuto +tyto +jej +zda +proč +máte +tato +kam +tohoto +kdo +kteří +mi +nám +tom +tomuto +mít +nic +proto +kterou +byla +toho +protože +asi +ho +naši +napište +re +což +tím +takže +svých +její +svými +jste +aj +tu +tedy +teto +bylo +kde +ke +pravé +ji +nad +nejsou +či +pod +téma +mezi +přes +ty +pak +vám +ani +když +však +neg +jsem +tento +článku +články +aby +jsme +před +pta +jejich +byl +ještě +až +bez +také +pouze +první +vaše +která +nás +nový +tipy +pokud +může +strana +jeho +své +jiné +zprávy +nové +není +vás +jen +podle +zde +už +být +více +bude +již +než +který +by +které +co +nebo +ten +tak +má +při +od +po +jsou +jak +další +ale +si +se +ve +to +jako +za +zpět +ze +do +pro +je +na +atd +atp +jakmile +přičemž +já +on +ona +ono +oni +ony +my +vy +jí +ji +mě +mne +jemu +tomu +těm +těmu +němu +němuž +jehož +jíž +jelikož +jež +jakož +načež diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_da.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_da.txt new file mode 100644 index 000000000..42e6145b9 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_da.txt @@ -0,0 +1,110 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/danish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Danish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + +og | and +i | in +jeg | I +det | that (dem. pronoun)/it (pers. pronoun) +at | that (in front of a sentence)/to (with infinitive) +en | a/an +den | it (pers. pronoun)/that (dem. pronoun) +til | to/at/for/until/against/by/of/into, more +er | present tense of "to be" +som | who, as +på | on/upon/in/on/at/to/after/of/with/for, on +de | they +med | with/by/in, along +han | he +af | of/by/from/off/for/in/with/on, off +for | at/for/to/from/by/of/ago, in front/before, because +ikke | not +der | who/which, there/those +var | past tense of "to be" +mig | me/myself +sig | oneself/himself/herself/itself/themselves +men | but +et | a/an/one, one (number), someone/somebody/one +har | present tense of "to have" +om | round/about/for/in/a, about/around/down, if +vi | we +min | my +havde | past tense of "to have" +ham | him +hun | she +nu | now +over | over/above/across/by/beyond/past/on/about, over/past +da | then, when/as/since +fra | from/off/since, off, since +du | you +ud | out +sin | his/her/its/one's +dem | them +os | us/ourselves +op | up +man | you/one +hans | his +hvor | where +eller | or +hvad | what +skal | must/shall etc. +selv | myself/youself/herself/ourselves etc., even +her | here +alle | all/everyone/everybody etc. +vil | will (verb) +blev | past tense of "to stay/to remain/to get/to become" +kunne | could +ind | in +når | when +være | present tense of "to be" +dog | however/yet/after all +noget | something +ville | would +jo | you know/you see (adv), yes +deres | their/theirs +efter | after/behind/according to/for/by/from, later/afterwards +ned | down +skulle | should +denne | this +end | than +dette | this +mit | my/mine +også | also +under | under/beneath/below/during, below/underneath +have | have +dig | you +anden | other +hende | her +mine | my +alt | everything +meget | much/very, plenty of +sit | his, her, its, one's +sine | his, her, its, one's +vor | our +mod | against +disse | these +hvis | if +din | your/yours +nogle | some +hos | by/at +blive | be/become +mange | many +ad | by/through +bliver | present tense of "to be/to become" +hendes | her/hers +været | be +thi | for (conj) +jer | you +sådan | such, like this/like that diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_de.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_de.txt new file mode 100644 index 000000000..86525e7ae --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_de.txt @@ -0,0 +1,294 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A German stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | The number of forms in this list is reduced significantly by passing it + | through the German stemmer. + + +aber | but + +alle | all +allem +allen +aller +alles + +als | than, as +also | so +am | an + dem +an | at + +ander | other +andere +anderem +anderen +anderer +anderes +anderm +andern +anderr +anders + +auch | also +auf | on +aus | out of +bei | by +bin | am +bis | until +bist | art +da | there +damit | with it +dann | then + +der | the +den +des +dem +die +das + +daß | that + +derselbe | the same +derselben +denselben +desselben +demselben +dieselbe +dieselben +dasselbe + +dazu | to that + +dein | thy +deine +deinem +deinen +deiner +deines + +denn | because + +derer | of those +dessen | of him + +dich | thee +dir | to thee +du | thou + +dies | this +diese +diesem +diesen +dieser +dieses + + +doch | (several meanings) +dort | (over) there + + +durch | through + +ein | a +eine +einem +einen +einer +eines + +einig | some +einige +einigem +einigen +einiger +einiges + +einmal | once + +er | he +ihn | him +ihm | to him + +es | it +etwas | something + +euer | your +eure +eurem +euren +eurer +eures + +für | for +gegen | towards +gewesen | p.p. of sein +hab | have +habe | have +haben | have +hat | has +hatte | had +hatten | had +hier | here +hin | there +hinter | behind + +ich | I +mich | me +mir | to me + + +ihr | you, to her +ihre +ihrem +ihren +ihrer +ihres +euch | to you + +im | in + dem +in | in +indem | while +ins | in + das +ist | is + +jede | each, every +jedem +jeden +jeder +jedes + +jene | that +jenem +jenen +jener +jenes + +jetzt | now +kann | can + +kein | no +keine +keinem +keinen +keiner +keines + +können | can +könnte | could +machen | do +man | one + +manche | some, many a +manchem +manchen +mancher +manches + +mein | my +meine +meinem +meinen +meiner +meines + +mit | with +muss | must +musste | had to +nach | to(wards) +nicht | not +nichts | nothing +noch | still, yet +nun | now +nur | only +ob | whether +oder | or +ohne | without +sehr | very + +sein | his +seine +seinem +seinen +seiner +seines + +selbst | self +sich | herself + +sie | they, she +ihnen | to them + +sind | are +so | so + +solche | such +solchem +solchen +solcher +solches + +soll | shall +sollte | should +sondern | but +sonst | else +über | over +um | about, around +und | and + +uns | us +unse +unsem +unsen +unser +unses + +unter | under +viel | much +vom | von + dem +von | from +vor | before +während | while +war | was +waren | were +warst | wast +was | what +weg | away, off +weil | because +weiter | further + +welche | which +welchem +welchen +welcher +welches + +wenn | when +werde | will +werden | will +wie | how +wieder | again +will | want +wir | we +wird | will +wirst | willst +wo | where +wollen | want +wollte | wanted +würde | would +würden | would +zu | to +zum | zu + dem +zur | zu + der +zwar | indeed +zwischen | between + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_el.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_el.txt new file mode 100644 index 000000000..232681f5b --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_el.txt @@ -0,0 +1,78 @@ +# Lucene Greek Stopwords list +# Note: by default this file is used after GreekLowerCaseFilter, +# so when modifying this file use 'σ' instead of 'ς' +ο +η +το +οι +τα +του +τησ +των +τον +την +και +κι +κ +ειμαι +εισαι +ειναι +ειμαστε +ειστε +στο +στον +στη +στην +μα +αλλα +απο +για +προσ +με +σε +ωσ +παρα +αντι +κατα +μετα +θα +να +δε +δεν +μη +μην +επι +ενω +εαν +αν +τοτε +που +πωσ +ποιοσ +ποια +ποιο +ποιοι +ποιεσ +ποιων +ποιουσ +αυτοσ +αυτη +αυτο +αυτοι +αυτων +αυτουσ +αυτεσ +αυτα +εκεινοσ +εκεινη +εκεινο +εκεινοι +εκεινεσ +εκεινα +εκεινων +εκεινουσ +οπωσ +ομωσ +ισωσ +οσο +οτι diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_en.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_en.txt new file mode 100644 index 000000000..8bb2b7de4 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_en.txt @@ -0,0 +1,332 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +### Top 200 words based on frequency in common use + stemmed versions +### > 1000 per million words +the +of +and +a +in +to +it +is +was +wa +I +for +that +you +he +be +with +on +by +at +have +are +ar +not +this +thi +but +had +they +thei +his +hi +from + +### > 3000 PMW +she +which +or +we +an +were +as + +### > 2000 PMW +do +been +their +has +ha +would +there +what +will +all +if +can +her +#said +who + +### Top 50 +### > 1000 PMW +#one +#on +so +up +them +some +when +could +him +into +its +it +then +#two +out +#time +my +about +#did +#your +#now +me +no +other +only +onli +#just +more +these +also +#people +#peopl +#know +any +ani +#first +#see +very +veri +new +#may +#mai +#well +should +#like +than +how + +### Top 90 +### > 900 PMW +#get +#way +#wai +our +#made +#got +after +#think +between +#many +#mani +#years +#year + +### Top 100 +### > 800 PMW +er +those +#go +being +be +because +becaus +down +#yeah + +### > 700 PMW +#three +#good +#back +#make +such +#through +#year +#over +#must +#still +#even +#take +#too + +### Top 120 above + +### > 600 PMW +#here +#come +#own +#last +#does +#doe +#oh +#say +#sai +#work +#where +#erm +#us +#government +#govern +#same +#man +#might +#day +#dai +#yes +#ye +#however +#howev + +### > 500 PMW +#put +#world +#another +#anoth +#want +#life +#most +#against +#again +#never +#under +#old +#much +#something +#someth +#Mr +#why +#each +#while +#house +#hous + +### > 400 PMW +#part +#number +#out of +#found +#off +#different +#differ +#went +#really +#realli +#thought +#came +#used +#us +#children +#always +#alwai +#four +#without +#give +#few +#within +#system +#local +#place +#great +#during +#dure +#although +#small +#before +#befor +#look +#case +#next +#end +#things +#thing +#social +#find +#group +#quite +#quit +#mean +#five +#party +#parti +#company +#compani +#every +#everi + +### > 300 PMW +#women +#says +#sai +#important +#import +#took + +### Top 200 + + + + + + +###################### Lucene defaults .... + + +# Standard english stop words taken from Lucene's StopAnalyzer +# - Added stemmed varients - are -> ar, they -> thei, this -> thi, was -> wa +#a +#an +#and +#are +#ar +#as +#at +#be +#but +#by +#for +#if +#in +#into +#is +#it +#no +#not +#of +#on +#or +#such +#that +#the +#their +#then +#there +#these +#they +#thei +#this +#thi +#to +#was +#wa +#will +#with diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_es.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_es.txt new file mode 100644 index 000000000..487d78c8d --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_es.txt @@ -0,0 +1,356 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Spanish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + + | The following is a ranked list (commonest to rarest) of stopwords + | deriving from a large sample of text. + + | Extra words have been added at the end. + +de | from, of +la | the, her +que | who, that +el | the +en | in +y | and +a | to +los | the, them +del | de + el +se | himself, from him etc +las | the, them +por | for, by, etc +un | a +para | for +con | with +no | no +una | a +su | his, her +al | a + el + | es from SER +lo | him +como | how +más | more +pero | pero +sus | su plural +le | to him, her +ya | already +o | or + | fue from SER +este | this + | ha from HABER +sí | himself etc +porque | because +esta | this + | son from SER +entre | between + | está from ESTAR +cuando | when +muy | very +sin | without +sobre | on + | ser from SER + | tiene from TENER +también | also +me | me +hasta | until +hay | there is/are +donde | where + | han from HABER +quien | whom, that + | están from ESTAR + | estado from ESTAR +desde | from +todo | all +nos | us +durante | during + | estados from ESTAR +todos | all +uno | a +les | to them +ni | nor +contra | against +otros | other + | fueron from SER +ese | that +eso | that + | había from HABER +ante | before +ellos | they +e | and (variant of y) +esto | this +mí | me +antes | before +algunos | some +qué | what? +unos | a +yo | I +otro | other +otras | other +otra | other +él | he +tanto | so much, many +esa | that +estos | these +mucho | much, many +quienes | who +nada | nothing +muchos | many +cual | who + | sea from SER +poco | few +ella | she +estar | to be + | haber from HABER +estas | these + | estaba from ESTAR + | estamos from ESTAR +algunas | some +algo | something +nosotros | we + + | other forms + +mi | me +mis | mi plural +tú | thou +te | thee +ti | thee +tu | thy +tus | tu plural +ellas | they +nosotras | we +vosotros | you +vosotras | you +os | you +mío | mine +mía | +míos | +mías | +tuyo | thine +tuya | +tuyos | +tuyas | +suyo | his, hers, theirs +suya | +suyos | +suyas | +nuestro | ours +nuestra | +nuestros | +nuestras | +vuestro | yours +vuestra | +vuestros | +vuestras | +esos | those +esas | those + + | forms of estar, to be (not including the infinitive): +estoy +estás +está +estamos +estáis +están +esté +estés +estemos +estéis +estén +estaré +estarás +estará +estaremos +estaréis +estarán +estaría +estarías +estaríamos +estaríais +estarían +estaba +estabas +estábamos +estabais +estaban +estuve +estuviste +estuvo +estuvimos +estuvisteis +estuvieron +estuviera +estuvieras +estuviéramos +estuvierais +estuvieran +estuviese +estuvieses +estuviésemos +estuvieseis +estuviesen +estando +estado +estada +estados +estadas +estad + + | forms of haber, to have (not including the infinitive): +he +has +ha +hemos +habéis +han +haya +hayas +hayamos +hayáis +hayan +habré +habrás +habrá +habremos +habréis +habrán +habría +habrías +habríamos +habríais +habrían +había +habías +habíamos +habíais +habían +hube +hubiste +hubo +hubimos +hubisteis +hubieron +hubiera +hubieras +hubiéramos +hubierais +hubieran +hubiese +hubieses +hubiésemos +hubieseis +hubiesen +habiendo +habido +habida +habidos +habidas + + | forms of ser, to be (not including the infinitive): +soy +eres +es +somos +sois +son +sea +seas +seamos +seáis +sean +seré +serás +será +seremos +seréis +serán +sería +serías +seríamos +seríais +serían +era +eras +éramos +erais +eran +fui +fuiste +fue +fuimos +fuisteis +fueron +fuera +fueras +fuéramos +fuerais +fueran +fuese +fueses +fuésemos +fueseis +fuesen +siendo +sido + | sed also means 'thirst' + + | forms of tener, to have (not including the infinitive): +tengo +tienes +tiene +tenemos +tenéis +tienen +tenga +tengas +tengamos +tengáis +tengan +tendré +tendrás +tendrá +tendremos +tendréis +tendrán +tendría +tendrías +tendríamos +tendríais +tendrían +tenía +tenías +teníamos +teníais +tenían +tuve +tuviste +tuvo +tuvimos +tuvisteis +tuvieron +tuviera +tuvieras +tuviéramos +tuvierais +tuvieran +tuviese +tuvieses +tuviésemos +tuvieseis +tuviesen +teniendo +tenido +tenida +tenidos +tenidas +tened + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_eu.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_eu.txt new file mode 100644 index 000000000..25f1db934 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_eu.txt @@ -0,0 +1,99 @@ +# example set of basque stopwords +al +anitz +arabera +asko +baina +bat +batean +batek +bati +batzuei +batzuek +batzuetan +batzuk +bera +beraiek +berau +berauek +bere +berori +beroriek +beste +bezala +da +dago +dira +ditu +du +dute +edo +egin +ere +eta +eurak +ez +gainera +gu +gutxi +guzti +haiei +haiek +haietan +hainbeste +hala +han +handik +hango +hara +hari +hark +hartan +hau +hauei +hauek +hauetan +hemen +hemendik +hemengo +hi +hona +honek +honela +honetan +honi +hor +hori +horiei +horiek +horietan +horko +horra +horrek +horrela +horretan +horri +hortik +hura +izan +ni +noiz +nola +non +nondik +nongo +nor +nora +ze +zein +zen +zenbait +zenbat +zer +zergatik +ziren +zituen +zu +zuek +zuen +zuten diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fa.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fa.txt new file mode 100644 index 000000000..723641c6d --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fa.txt @@ -0,0 +1,313 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +# Note: by default this file is used after normalization, so when adding entries +# to this file, use the arabic 'ي' instead of 'ی' +انان +نداشته +سراسر +خياه +ايشان +وي +تاكنون +بيشتري +دوم +پس +ناشي +وگو +يا +داشتند +سپس +هنگام +هرگز +پنج +نشان +امسال +ديگر +گروهي +شدند +چطور +ده +و +دو +نخستين +ولي +چرا +چه +وسط +ه +كدام +قابل +يك +رفت +هفت +همچنين +در +هزار +بله +بلي +شايد +اما +شناسي +گرفته +دهد +داشته +دانست +داشتن +خواهيم +ميليارد +وقتيكه +امد +خواهد +جز +اورده +شده +بلكه +خدمات +شدن +برخي +نبود +بسياري +جلوگيري +حق +كردند +نوعي +بعري +نكرده +نظير +نبايد +بوده +بودن +داد +اورد +هست +جايي +شود +دنبال +داده +بايد +سابق +هيچ +همان +انجا +كمتر +كجاست +گردد +كسي +تر +مردم +تان +دادن +بودند +سري +جدا +ندارند +مگر +يكديگر +دارد +دهند +بنابراين +هنگامي +سمت +جا +انچه +خود +دادند +زياد +دارند +اثر +بدون +بهترين +بيشتر +البته +به +براساس +بيرون +كرد +بعضي +گرفت +توي +اي +ميليون +او +جريان +تول +بر +مانند +برابر +باشيم +مدتي +گويند +اكنون +تا +تنها +جديد +چند +بي +نشده +كردن +كردم +گويد +كرده +كنيم +نمي +نزد +روي +قصد +فقط +بالاي +ديگران +اين +ديروز +توسط +سوم +ايم +دانند +سوي +استفاده +شما +كنار +داريم +ساخته +طور +امده +رفته +نخست +بيست +نزديك +طي +كنيد +از +انها +تمامي +داشت +يكي +طريق +اش +چيست +روب +نمايد +گفت +چندين +چيزي +تواند +ام +ايا +با +ان +ايد +ترين +اينكه +ديگري +راه +هايي +بروز +همچنان +پاعين +كس +حدود +مختلف +مقابل +چيز +گيرد +ندارد +ضد +همچون +سازي +شان +مورد +باره +مرسي +خويش +برخوردار +چون +خارج +شش +هنوز +تحت +ضمن +هستيم +گفته +فكر +بسيار +پيش +براي +روزهاي +انكه +نخواهد +بالا +كل +وقتي +كي +چنين +كه +گيري +نيست +است +كجا +كند +نيز +يابد +بندي +حتي +توانند +عقب +خواست +كنند +بين +تمام +همه +ما +باشند +مثل +شد +اري +باشد +اره +طبق +بعد +اگر +صورت +غير +جاي +بيش +ريزي +اند +زيرا +چگونه +بار +لطفا +مي +درباره +من +ديده +همين +گذاري +برداري +علت +گذاشته +هم +فوق +نه +ها +شوند +اباد +همواره +هر +اول +خواهند +چهار +نام +امروز +مان +هاي +قبل +كنم +سعي +تازه +را +هستند +زير +جلوي +عنوان +بود diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fi.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fi.txt new file mode 100644 index 000000000..4372c9a05 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fi.txt @@ -0,0 +1,97 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + +| forms of BE + +olla +olen +olet +on +olemme +olette +ovat +ole | negative form + +oli +olisi +olisit +olisin +olisimme +olisitte +olisivat +olit +olin +olimme +olitte +olivat +ollut +olleet + +en | negation +et +ei +emme +ette +eivät + +|Nom Gen Acc Part Iness Elat Illat Adess Ablat Allat Ess Trans +minä minun minut minua minussa minusta minuun minulla minulta minulle | I +sinä sinun sinut sinua sinussa sinusta sinuun sinulla sinulta sinulle | you +hän hänen hänet häntä hänessä hänestä häneen hänellä häneltä hänelle | he she +me meidän meidät meitä meissä meistä meihin meillä meiltä meille | we +te teidän teidät teitä teissä teistä teihin teillä teiltä teille | you +he heidän heidät heitä heissä heistä heihin heillä heiltä heille | they + +tämä tämän tätä tässä tästä tähän tallä tältä tälle tänä täksi | this +tuo tuon tuotä tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that +se sen sitä siinä siitä siihen sillä siltä sille sinä siksi | it +nämä näiden näitä näissä näistä näihin näillä näiltä näille näinä näiksi | these +nuo noiden noita noissa noista noihin noilla noilta noille noina noiksi | those +ne niiden niitä niissä niistä niihin niillä niiltä niille niinä niiksi | they + +kuka kenen kenet ketä kenessä kenestä keneen kenellä keneltä kenelle kenenä keneksi| who +ketkä keiden ketkä keitä keissä keistä keihin keillä keiltä keille keinä keiksi | (pl) +mikä minkä minkä mitä missä mistä mihin millä miltä mille minä miksi | which what +mitkä | (pl) + +joka jonka jota jossa josta johon jolla jolta jolle jona joksi | who which +jotka joiden joita joissa joista joihin joilla joilta joille joina joiksi | (pl) + +| conjunctions + +että | that +ja | and +jos | if +koska | because +kuin | than +mutta | but +niin | so +sekä | and +sillä | for +tai | or +vaan | but +vai | or +vaikka | although + + +| prepositions + +kanssa | with +mukaan | according to +noin | about +poikki | across +yli | over, across + +| other + +kun | when +niin | so +nyt | now +itse | self + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fr.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fr.txt new file mode 100644 index 000000000..749abae68 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fr.txt @@ -0,0 +1,186 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A French stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + +au | a + le +aux | a + les +avec | with +ce | this +ces | these +dans | with +de | of +des | de + les +du | de + le +elle | she +en | `of them' etc +et | and +eux | them +il | he +je | I +la | the +le | the +leur | their +lui | him +ma | my (fem) +mais | but +me | me +même | same; as in moi-même (myself) etc +mes | me (pl) +moi | me +mon | my (masc) +ne | not +nos | our (pl) +notre | our +nous | we +on | one +ou | where +par | by +pas | not +pour | for +qu | que before vowel +que | that +qui | who +sa | his, her (fem) +se | oneself +ses | his (pl) +son | his, her (masc) +sur | on +ta | thy (fem) +te | thee +tes | thy (pl) +toi | thee +ton | thy (masc) +tu | thou +un | a +une | a +vos | your (pl) +votre | your +vous | you + + | single letter forms + +c | c' +d | d' +j | j' +l | l' +à | to, at +m | m' +n | n' +s | s' +t | t' +y | there + + | forms of être (not including the infinitive): +été +étée +étées +étés +étant +suis +es +est +sommes +êtes +sont +serai +seras +sera +serons +serez +seront +serais +serait +serions +seriez +seraient +étais +était +étions +étiez +étaient +fus +fut +fûmes +fûtes +furent +sois +soit +soyons +soyez +soient +fusse +fusses +fût +fussions +fussiez +fussent + + | forms of avoir (not including the infinitive): +ayant +eu +eue +eues +eus +ai +as +avons +avez +ont +aurai +auras +aura +aurons +aurez +auront +aurais +aurait +aurions +auriez +auraient +avais +avait +avions +aviez +avaient +eut +eûmes +eûtes +eurent +aie +aies +ait +ayons +ayez +aient +eusse +eusses +eût +eussions +eussiez +eussent + + | Later additions (from Jean-Christophe Deschamps) +ceci | this +cela | that +celà | that +cet | this +cette | this +ici | here +ils | they +les | the (pl) +leurs | their (pl) +quel | which +quels | which +quelle | which +quelles | which +sans | without +soi | oneself + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ga.txt new file mode 100644 index 000000000..9ff88d747 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ga.txt @@ -0,0 +1,110 @@ + +a +ach +ag +agus +an +aon +ar +arna +as +b' +ba +beirt +bhúr +caoga +ceathair +ceathrar +chomh +chtó +chuig +chun +cois +céad +cúig +cúigear +d' +daichead +dar +de +deich +deichniúr +den +dhá +do +don +dtí +dá +dár +dó +faoi +faoin +faoina +faoinár +fara +fiche +gach +gan +go +gur +haon +hocht +i +iad +idir +in +ina +ins +inár +is +le +leis +lena +lenár +m' +mar +mo +mé +na +nach +naoi +naonúr +ná +ní +níor +nó +nócha +ocht +ochtar +os +roimh +sa +seacht +seachtar +seachtó +seasca +seisear +siad +sibh +sinn +sna +sé +sí +tar +thar +thú +triúr +trí +trína +trínár +tríocha +tú +um +ár +é +éis +í +ó +ón +óna +ónár diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_gl.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_gl.txt new file mode 100644 index 000000000..d8760b12c --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_gl.txt @@ -0,0 +1,161 @@ +# galican stopwords +a +aínda +alí +aquel +aquela +aquelas +aqueles +aquilo +aquí +ao +aos +as +así +á +ben +cando +che +co +coa +comigo +con +connosco +contigo +convosco +coas +cos +cun +cuns +cunha +cunhas +da +dalgunha +dalgunhas +dalgún +dalgúns +das +de +del +dela +delas +deles +desde +deste +do +dos +dun +duns +dunha +dunhas +e +el +ela +elas +eles +en +era +eran +esa +esas +ese +eses +esta +estar +estaba +está +están +este +estes +estiven +estou +eu +é +facer +foi +foron +fun +había +hai +iso +isto +la +las +lle +lles +lo +los +mais +me +meu +meus +min +miña +miñas +moi +na +nas +neste +nin +no +non +nos +nosa +nosas +noso +nosos +nós +nun +nunha +nuns +nunhas +o +os +ou +ó +ós +para +pero +pode +pois +pola +polas +polo +polos +por +que +se +senón +ser +seu +seus +sexa +sido +sobre +súa +súas +tamén +tan +te +ten +teñen +teño +ter +teu +teus +ti +tido +tiña +tiven +túa +túas +un +unha +unhas +uns +vos +vosa +vosas +voso +vosos +vós diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hi.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hi.txt new file mode 100644 index 000000000..86286bb08 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hi.txt @@ -0,0 +1,235 @@ +# Also see http://www.opensource.org/licenses/bsd-license.html +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# This file was created by Jacques Savoy and is distributed under the BSD license. +# Note: by default this file also contains forms normalized by HindiNormalizer +# for spelling variation (see section below), such that it can be used whether or +# not you enable that feature. When adding additional entries to this list, +# please add the normalized form as well. +अंदर +अत +अपना +अपनी +अपने +अभी +आदि +आप +इत्यादि +इन +इनका +इन्हीं +इन्हें +इन्हों +इस +इसका +इसकी +इसके +इसमें +इसी +इसे +उन +उनका +उनकी +उनके +उनको +उन्हीं +उन्हें +उन्हों +उस +उसके +उसी +उसे +एक +एवं +एस +ऐसे +और +कई +कर +करता +करते +करना +करने +करें +कहते +कहा +का +काफ़ी +कि +कितना +किन्हें +किन्हों +किया +किर +किस +किसी +किसे +की +कुछ +कुल +के +को +कोई +कौन +कौनसा +गया +घर +जब +जहाँ +जा +जितना +जिन +जिन्हें +जिन्हों +जिस +जिसे +जीधर +जैसा +जैसे +जो +तक +तब +तरह +तिन +तिन्हें +तिन्हों +तिस +तिसे +तो +था +थी +थे +दबारा +दिया +दुसरा +दूसरे +दो +द्वारा +न +नहीं +ना +निहायत +नीचे +ने +पर +पर +पहले +पूरा +पे +फिर +बनी +बही +बहुत +बाद +बाला +बिलकुल +भी +भीतर +मगर +मानो +मे +में +यदि +यह +यहाँ +यही +या +यिह +ये +रखें +रहा +रहे +ऱ्वासा +लिए +लिये +लेकिन +व +वर्ग +वह +वह +वहाँ +वहीं +वाले +वुह +वे +वग़ैरह +संग +सकता +सकते +सबसे +सभी +साथ +साबुत +साभ +सारा +से +सो +ही +हुआ +हुई +हुए +है +हैं +हो +होता +होती +होते +होना +होने +# additional normalized forms of the above +अपनि +जेसे +होति +सभि +तिंहों +इंहों +दवारा +इसि +किंहें +थि +उंहों +ओर +जिंहें +वहिं +अभि +बनि +हि +उंहिं +उंहें +हें +वगेरह +एसे +रवासा +कोन +निचे +काफि +उसि +पुरा +भितर +हे +बहि +वहां +कोइ +यहां +जिंहों +तिंहें +किसि +कइ +यहि +इंहिं +जिधर +इंहें +अदि +इतयादि +हुइ +कोनसा +इसकि +दुसरे +जहां +अप +किंहों +उनकि +भि +वरग +हुअ +जेसा +नहिं diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hu.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hu.txt new file mode 100644 index 000000000..37526da8a --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hu.txt @@ -0,0 +1,211 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/hungarian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + +| Hungarian stop word list +| prepared by Anna Tordai + +a +ahogy +ahol +aki +akik +akkor +alatt +által +általában +amely +amelyek +amelyekben +amelyeket +amelyet +amelynek +ami +amit +amolyan +amíg +amikor +át +abban +ahhoz +annak +arra +arról +az +azok +azon +azt +azzal +azért +aztán +azután +azonban +bár +be +belül +benne +cikk +cikkek +cikkeket +csak +de +e +eddig +egész +egy +egyes +egyetlen +egyéb +egyik +egyre +ekkor +el +elég +ellen +elő +először +előtt +első +én +éppen +ebben +ehhez +emilyen +ennek +erre +ez +ezt +ezek +ezen +ezzel +ezért +és +fel +felé +hanem +hiszen +hogy +hogyan +igen +így +illetve +ill. +ill +ilyen +ilyenkor +ison +ismét +itt +jó +jól +jobban +kell +kellett +keresztül +keressünk +ki +kívül +között +közül +legalább +lehet +lehetett +legyen +lenne +lenni +lesz +lett +maga +magát +majd +majd +már +más +másik +meg +még +mellett +mert +mely +melyek +mi +mit +míg +miért +milyen +mikor +minden +mindent +mindenki +mindig +mint +mintha +mivel +most +nagy +nagyobb +nagyon +ne +néha +nekem +neki +nem +néhány +nélkül +nincs +olyan +ott +össze +ő +ők +őket +pedig +persze +rá +s +saját +sem +semmi +sok +sokat +sokkal +számára +szemben +szerint +szinte +talán +tehát +teljes +tovább +továbbá +több +úgy +ugyanis +új +újabb +újra +után +utána +utolsó +vagy +vagyis +valaki +valami +valamint +való +vagyok +van +vannak +volt +voltam +voltak +voltunk +vissza +vele +viszont +volna diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hy.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hy.txt new file mode 100644 index 000000000..60c1c50fb --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hy.txt @@ -0,0 +1,46 @@ +# example set of Armenian stopwords. +այդ +այլ +այն +այս +դու +դուք +եմ +են +ենք +ես +եք +է +էի +էին +էինք +էիր +էիք +էր +ըստ +թ +ի +ին +իսկ +իր +կամ +համար +հետ +հետո +մենք +մեջ +մի +ն +նա +նաև +նրա +նրանք +որ +որը +որոնք +որպես +ու +ում +պիտի +վրա +և diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_id.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_id.txt new file mode 100644 index 000000000..4617f83a5 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_id.txt @@ -0,0 +1,359 @@ +# from appendix D of: A Study of Stemming Effects on Information +# Retrieval in Bahasa Indonesia +ada +adanya +adalah +adapun +agak +agaknya +agar +akan +akankah +akhirnya +aku +akulah +amat +amatlah +anda +andalah +antar +diantaranya +antara +antaranya +diantara +apa +apaan +mengapa +apabila +apakah +apalagi +apatah +atau +ataukah +ataupun +bagai +bagaikan +sebagai +sebagainya +bagaimana +bagaimanapun +sebagaimana +bagaimanakah +bagi +bahkan +bahwa +bahwasanya +sebaliknya +banyak +sebanyak +beberapa +seberapa +begini +beginian +beginikah +beginilah +sebegini +begitu +begitukah +begitulah +begitupun +sebegitu +belum +belumlah +sebelum +sebelumnya +sebenarnya +berapa +berapakah +berapalah +berapapun +betulkah +sebetulnya +biasa +biasanya +bila +bilakah +bisa +bisakah +sebisanya +boleh +bolehkah +bolehlah +buat +bukan +bukankah +bukanlah +bukannya +cuma +percuma +dahulu +dalam +dan +dapat +dari +daripada +dekat +demi +demikian +demikianlah +sedemikian +dengan +depan +di +dia +dialah +dini +diri +dirinya +terdiri +dong +dulu +enggak +enggaknya +entah +entahlah +terhadap +terhadapnya +hal +hampir +hanya +hanyalah +harus +haruslah +harusnya +seharusnya +hendak +hendaklah +hendaknya +hingga +sehingga +ia +ialah +ibarat +ingin +inginkah +inginkan +ini +inikah +inilah +itu +itukah +itulah +jangan +jangankan +janganlah +jika +jikalau +juga +justru +kala +kalau +kalaulah +kalaupun +kalian +kami +kamilah +kamu +kamulah +kan +kapan +kapankah +kapanpun +dikarenakan +karena +karenanya +ke +kecil +kemudian +kenapa +kepada +kepadanya +ketika +seketika +khususnya +kini +kinilah +kiranya +sekiranya +kita +kitalah +kok +lagi +lagian +selagi +lah +lain +lainnya +melainkan +selaku +lalu +melalui +terlalu +lama +lamanya +selama +selama +selamanya +lebih +terlebih +bermacam +macam +semacam +maka +makanya +makin +malah +malahan +mampu +mampukah +mana +manakala +manalagi +masih +masihkah +semasih +masing +mau +maupun +semaunya +memang +mereka +merekalah +meski +meskipun +semula +mungkin +mungkinkah +nah +namun +nanti +nantinya +nyaris +oleh +olehnya +seorang +seseorang +pada +padanya +padahal +paling +sepanjang +pantas +sepantasnya +sepantasnyalah +para +pasti +pastilah +per +pernah +pula +pun +merupakan +rupanya +serupa +saat +saatnya +sesaat +saja +sajalah +saling +bersama +sama +sesama +sambil +sampai +sana +sangat +sangatlah +saya +sayalah +se +sebab +sebabnya +sebuah +tersebut +tersebutlah +sedang +sedangkan +sedikit +sedikitnya +segala +segalanya +segera +sesegera +sejak +sejenak +sekali +sekalian +sekalipun +sesekali +sekaligus +sekarang +sekarang +sekitar +sekitarnya +sela +selain +selalu +seluruh +seluruhnya +semakin +sementara +sempat +semua +semuanya +sendiri +sendirinya +seolah +seperti +sepertinya +sering +seringnya +serta +siapa +siapakah +siapapun +disini +disinilah +sini +sinilah +sesuatu +sesuatunya +suatu +sesudah +sesudahnya +sudah +sudahkah +sudahlah +supaya +tadi +tadinya +tak +tanpa +setelah +telah +tentang +tentu +tentulah +tentunya +tertentu +seterusnya +tapi +tetapi +setiap +tiap +setidaknya +tidak +tidakkah +tidaklah +toh +waduh +wah +wahai +sewaktu +walau +walaupun +wong +yaitu +yakni +yang diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_it.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_it.txt new file mode 100644 index 000000000..1219cc773 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_it.txt @@ -0,0 +1,303 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | An Italian stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + +ad | a (to) before vowel +al | a + il +allo | a + lo +ai | a + i +agli | a + gli +all | a + l' +agl | a + gl' +alla | a + la +alle | a + le +con | with +col | con + il +coi | con + i (forms collo, cogli etc are now very rare) +da | from +dal | da + il +dallo | da + lo +dai | da + i +dagli | da + gli +dall | da + l' +dagl | da + gll' +dalla | da + la +dalle | da + le +di | of +del | di + il +dello | di + lo +dei | di + i +degli | di + gli +dell | di + l' +degl | di + gl' +della | di + la +delle | di + le +in | in +nel | in + el +nello | in + lo +nei | in + i +negli | in + gli +nell | in + l' +negl | in + gl' +nella | in + la +nelle | in + le +su | on +sul | su + il +sullo | su + lo +sui | su + i +sugli | su + gli +sull | su + l' +sugl | su + gl' +sulla | su + la +sulle | su + le +per | through, by +tra | among +contro | against +io | I +tu | thou +lui | he +lei | she +noi | we +voi | you +loro | they +mio | my +mia | +miei | +mie | +tuo | +tua | +tuoi | thy +tue | +suo | +sua | +suoi | his, her +sue | +nostro | our +nostra | +nostri | +nostre | +vostro | your +vostra | +vostri | +vostre | +mi | me +ti | thee +ci | us, there +vi | you, there +lo | him, the +la | her, the +li | them +le | them, the +gli | to him, the +ne | from there etc +il | the +un | a +uno | a +una | a +ma | but +ed | and +se | if +perché | why, because +anche | also +come | how +dov | where (as dov') +dove | where +che | who, that +chi | who +cui | whom +non | not +più | more +quale | who, that +quanto | how much +quanti | +quanta | +quante | +quello | that +quelli | +quella | +quelle | +questo | this +questi | +questa | +queste | +si | yes +tutto | all +tutti | all + + | single letter forms: + +a | at +c | as c' for ce or ci +e | and +i | the +l | as l' +o | or + + | forms of avere, to have (not including the infinitive): + +ho +hai +ha +abbiamo +avete +hanno +abbia +abbiate +abbiano +avrò +avrai +avrà +avremo +avrete +avranno +avrei +avresti +avrebbe +avremmo +avreste +avrebbero +avevo +avevi +aveva +avevamo +avevate +avevano +ebbi +avesti +ebbe +avemmo +aveste +ebbero +avessi +avesse +avessimo +avessero +avendo +avuto +avuta +avuti +avute + + | forms of essere, to be (not including the infinitive): +sono +sei +è +siamo +siete +sia +siate +siano +sarò +sarai +sarà +saremo +sarete +saranno +sarei +saresti +sarebbe +saremmo +sareste +sarebbero +ero +eri +era +eravamo +eravate +erano +fui +fosti +fu +fummo +foste +furono +fossi +fosse +fossimo +fossero +essendo + + | forms of fare, to do (not including the infinitive, fa, fat-): +faccio +fai +facciamo +fanno +faccia +facciate +facciano +farò +farai +farà +faremo +farete +faranno +farei +faresti +farebbe +faremmo +fareste +farebbero +facevo +facevi +faceva +facevamo +facevate +facevano +feci +facesti +fece +facemmo +faceste +fecero +facessi +facesse +facessimo +facessero +facendo + + | forms of stare, to be (not including the infinitive): +sto +stai +sta +stiamo +stanno +stia +stiate +stiano +starò +starai +starà +staremo +starete +staranno +starei +staresti +starebbe +staremmo +stareste +starebbero +stavo +stavi +stava +stavamo +stavate +stavano +stetti +stesti +stette +stemmo +steste +stettero +stessi +stesse +stessimo +stessero +stando diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ja.txt new file mode 100644 index 000000000..d4321be6b --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ja.txt @@ -0,0 +1,127 @@ +# +# This file defines a stopword set for Japanese. +# +# This set is made up of hand-picked frequent terms from segmented Japanese Wikipedia. +# Punctuation characters and frequent kanji have mostly been left out. See LUCENE-3745 +# for frequency lists, etc. that can be useful for making your own set (if desired) +# +# Note that there is an overlap between these stopwords and the terms stopped when used +# in combination with the JapanesePartOfSpeechStopFilter. When editing this file, note +# that comments are not allowed on the same line as stopwords. +# +# Also note that stopping is done in a case-insensitive manner. Change your StopFilter +# configuration if you need case-sensitive stopping. Lastly, note that stopping is done +# using the same character width as the entries in this file. Since this StopFilter is +# normally done after a CJKWidthFilter in your chain, you would usually want your romaji +# entries to be in half-width and your kana entries to be in full-width. +# +の +に +は +を +た +が +で +て +と +し +れ +さ +ある +いる +も +する +から +な +こと +として +い +や +れる +など +なっ +ない +この +ため +その +あっ +よう +また +もの +という +あり +まで +られ +なる +へ +か +だ +これ +によって +により +おり +より +による +ず +なり +られる +において +ば +なかっ +なく +しかし +について +せ +だっ +その後 +できる +それ +う +ので +なお +のみ +でき +き +つ +における +および +いう +さらに +でも +ら +たり +その他 +に関する +たち +ます +ん +なら +に対して +特に +せる +及び +これら +とき +では +にて +ほか +ながら +うち +そして +とともに +ただし +かつて +それぞれ +または +お +ほど +ものの +に対する +ほとんど +と共に +といった +です +とも +ところ +ここ +##### End of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_lv.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_lv.txt new file mode 100644 index 000000000..e21a23c06 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_lv.txt @@ -0,0 +1,172 @@ +# Set of Latvian stopwords from A Stemming Algorithm for Latvian, Karlis Kreslins +# the original list of over 800 forms was refined: +# pronouns, adverbs, interjections were removed +# +# prepositions +aiz +ap +ar +apakš +ārpus +augšpus +bez +caur +dēļ +gar +iekš +iz +kopš +labad +lejpus +līdz +no +otrpus +pa +par +pār +pēc +pie +pirms +pret +priekš +starp +šaipus +uz +viņpus +virs +virspus +zem +apakšpus +# Conjunctions +un +bet +jo +ja +ka +lai +tomēr +tikko +turpretī +arī +kaut +gan +tādēļ +tā +ne +tikvien +vien +kā +ir +te +vai +kamēr +# Particles +ar +diezin +droši +diemžēl +nebūt +ik +it +taču +nu +pat +tiklab +iekšpus +nedz +tik +nevis +turpretim +jeb +iekam +iekām +iekāms +kolīdz +līdzko +tiklīdz +jebšu +tālab +tāpēc +nekā +itin +jā +jau +jel +nē +nezin +tad +tikai +vis +tak +iekams +vien +# modal verbs +būt +biju +biji +bija +bijām +bijāt +esmu +esi +esam +esat +būšu +būsi +būs +būsim +būsiet +tikt +tiku +tiki +tika +tikām +tikāt +tieku +tiec +tiek +tiekam +tiekat +tikšu +tiks +tiksim +tiksiet +tapt +tapi +tapāt +topat +tapšu +tapsi +taps +tapsim +tapsiet +kļūt +kļuvu +kļuvi +kļuva +kļuvām +kļuvāt +kļūstu +kļūsti +kļūst +kļūstam +kļūstat +kļūšu +kļūsi +kļūs +kļūsim +kļūsiet +# verbs +varēt +varēju +varējām +varēšu +varēsim +var +varēji +varējāt +varēsi +varēsiet +varat +varēja +varēs diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_nl.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_nl.txt new file mode 100644 index 000000000..47a2aeacf --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_nl.txt @@ -0,0 +1,119 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Dutch stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large sample of Dutch text. + + | Dutch stop words frequently exhibit homonym clashes. These are indicated + | clearly below. + +de | the +en | and +van | of, from +ik | I, the ego +te | (1) chez, at etc, (2) to, (3) too +dat | that, which +die | that, those, who, which +in | in, inside +een | a, an, one +hij | he +het | the, it +niet | not, nothing, naught +zijn | (1) to be, being, (2) his, one's, its +is | is +was | (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river +op | on, upon, at, in, up, used up +aan | on, upon, to (as dative) +met | with, by +als | like, such as, when +voor | (1) before, in front of, (2) furrow +had | had, past tense all persons sing. of 'hebben' (have) +er | there +maar | but, only +om | round, about, for etc +hem | him +dan | then +zou | should/would, past tense all persons sing. of 'zullen' +of | or, whether, if +wat | what, something, anything +mijn | possessive and noun 'mine' +men | people, 'one' +dit | this +zo | so, thus, in this way +door | through by +over | over, across +ze | she, her, they, them +zich | oneself +bij | (1) a bee, (2) by, near, at +ook | also, too +tot | till, until +je | you +mij | me +uit | out of, from +der | Old Dutch form of 'van der' still found in surnames +daar | (1) there, (2) because +haar | (1) her, their, them, (2) hair +naar | (1) unpleasant, unwell etc, (2) towards, (3) as +heb | present first person sing. of 'to have' +hoe | how, why +heeft | present third person sing. of 'to have' +hebben | 'to have' and various parts thereof +deze | this +u | you +want | (1) for, (2) mitten, (3) rigging +nog | yet, still +zal | 'shall', first and third person sing. of verb 'zullen' (will) +me | me +zij | she, they +nu | now +ge | 'thou', still used in Belgium and south Netherlands +geen | none +omdat | because +iets | something, somewhat +worden | to become, grow, get +toch | yet, still +al | all, every, each +waren | (1) 'were' (2) to wander, (3) wares, (3) +veel | much, many +meer | (1) more, (2) lake +doen | to do, to make +toen | then, when +moet | noun 'spot/mote' and present form of 'to must' +ben | (1) am, (2) 'are' in interrogative second person singular of 'to be' +zonder | without +kan | noun 'can' and present form of 'to be able' +hun | their, them +dus | so, consequently +alles | all, everything, anything +onder | under, beneath +ja | yes, of course +eens | once, one day +hier | here +wie | who +werd | imperfect third person sing. of 'become' +altijd | always +doch | yet, but etc +wordt | present third person sing. of 'become' +wezen | (1) to be, (2) 'been' as in 'been fishing', (3) orphans +kunnen | to be able +ons | us/our +zelf | self +tegen | against, towards, at +na | after, near +reeds | already +wil | (1) present tense of 'want', (2) 'will', noun, (3) fender +kon | could; past tense of 'to be able' +niets | nothing +uw | your +iemand | somebody +geweest | been; past participle of 'be' +andere | other diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_no.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_no.txt new file mode 100644 index 000000000..a7a2c28ba --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_no.txt @@ -0,0 +1,194 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/norwegian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Norwegian stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This stop word list is for the dominant bokmål dialect. Words unique + | to nynorsk are marked *. + + | Revised by Jan Bruusgaard , Jan 2005 + +og | and +i | in +jeg | I +det | it/this/that +at | to (w. inf.) +en | a/an +et | a/an +den | it/this/that +til | to +er | is/am/are +som | who/that +på | on +de | they / you(formal) +med | with +han | he +av | of +ikke | not +ikkje | not * +der | there +så | so +var | was/were +meg | me +seg | you +men | but +ett | one +har | have +om | about +vi | we +min | my +mitt | my +ha | have +hadde | had +hun | she +nå | now +over | over +da | when/as +ved | by/know +fra | from +du | you +ut | out +sin | your +dem | them +oss | us +opp | up +man | you/one +kan | can +hans | his +hvor | where +eller | or +hva | what +skal | shall/must +selv | self (reflective) +sjøl | self (reflective) +her | here +alle | all +vil | will +bli | become +ble | became +blei | became * +blitt | have become +kunne | could +inn | in +når | when +være | be +kom | come +noen | some +noe | some +ville | would +dere | you +som | who/which/that +deres | their/theirs +kun | only/just +ja | yes +etter | after +ned | down +skulle | should +denne | this +for | for/because +deg | you +si | hers/his +sine | hers/his +sitt | hers/his +mot | against +å | to +meget | much +hvorfor | why +dette | this +disse | these/those +uten | without +hvordan | how +ingen | none +din | your +ditt | your +blir | become +samme | same +hvilken | which +hvilke | which (plural) +sånn | such a +inni | inside/within +mellom | between +vår | our +hver | each +hvem | who +vors | us/ours +hvis | whose +både | both +bare | only/just +enn | than +fordi | as/because +før | before +mange | many +også | also +slik | just +vært | been +være | to be +båe | both * +begge | both +siden | since +dykk | your * +dykkar | yours * +dei | they * +deira | them * +deires | theirs * +deim | them * +di | your (fem.) * +då | as/when * +eg | I * +ein | a/an * +eit | a/an * +eitt | a/an * +elles | or * +honom | he * +hjå | at * +ho | she * +hoe | she * +henne | her +hennar | her/hers +hennes | hers +hoss | how * +hossen | how * +ikkje | not * +ingi | noone * +inkje | noone * +korleis | how * +korso | how * +kva | what/which * +kvar | where * +kvarhelst | where * +kven | who/whom * +kvi | why * +kvifor | why * +me | we * +medan | while * +mi | my * +mine | my * +mykje | much * +no | now * +nokon | some (masc./neut.) * +noka | some (fem.) * +nokor | some * +noko | some * +nokre | some * +si | his/hers * +sia | since * +sidan | since * +so | so * +somt | some * +somme | some * +um | about* +upp | up * +vere | be * +vore | was * +verte | become * +vort | become * +varte | became * +vart | became * + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_pt.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_pt.txt new file mode 100644 index 000000000..acfeb01af --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_pt.txt @@ -0,0 +1,253 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Portuguese stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + + | The following is a ranked list (commonest to rarest) of stopwords + | deriving from a large sample of text. + + | Extra words have been added at the end. + +de | of, from +a | the; to, at; her +o | the; him +que | who, that +e | and +do | de + o +da | de + a +em | in +um | a +para | for + | é from SER +com | with +não | not, no +uma | a +os | the; them +no | em + o +se | himself etc +na | em + a +por | for +mais | more +as | the; them +dos | de + os +como | as, like +mas | but + | foi from SER +ao | a + o +ele | he +das | de + as + | tem from TER +à | a + a +seu | his +sua | her +ou | or + | ser from SER +quando | when +muito | much + | há from HAV +nos | em + os; us +já | already, now + | está from EST +eu | I +também | also +só | only, just +pelo | per + o +pela | per + a +até | up to +isso | that +ela | he +entre | between + | era from SER +depois | after +sem | without +mesmo | same +aos | a + os + | ter from TER +seus | his +quem | whom +nas | em + as +me | me +esse | that +eles | they + | estão from EST +você | you + | tinha from TER + | foram from SER +essa | that +num | em + um +nem | nor +suas | her +meu | my +às | a + as +minha | my + | têm from TER +numa | em + uma +pelos | per + os +elas | they + | havia from HAV + | seja from SER +qual | which + | será from SER +nós | we + | tenho from TER +lhe | to him, her +deles | of them +essas | those +esses | those +pelas | per + as +este | this + | fosse from SER +dele | of him + + | other words. There are many contractions such as naquele = em+aquele, + | mo = me+o, but they are rare. + | Indefinite article plural forms are also rare. + +tu | thou +te | thee +vocês | you (plural) +vos | you +lhes | to them +meus | my +minhas +teu | thy +tua +teus +tuas +nosso | our +nossa +nossos +nossas + +dela | of her +delas | of them + +esta | this +estes | these +estas | these +aquele | that +aquela | that +aqueles | those +aquelas | those +isto | this +aquilo | that + + | forms of estar, to be (not including the infinitive): +estou +está +estamos +estão +estive +esteve +estivemos +estiveram +estava +estávamos +estavam +estivera +estivéramos +esteja +estejamos +estejam +estivesse +estivéssemos +estivessem +estiver +estivermos +estiverem + + | forms of haver, to have (not including the infinitive): +hei +há +havemos +hão +houve +houvemos +houveram +houvera +houvéramos +haja +hajamos +hajam +houvesse +houvéssemos +houvessem +houver +houvermos +houverem +houverei +houverá +houveremos +houverão +houveria +houveríamos +houveriam + + | forms of ser, to be (not including the infinitive): +sou +somos +são +era +éramos +eram +fui +foi +fomos +foram +fora +fôramos +seja +sejamos +sejam +fosse +fôssemos +fossem +for +formos +forem +serei +será +seremos +serão +seria +seríamos +seriam + + | forms of ter, to have (not including the infinitive): +tenho +tem +temos +tém +tinha +tínhamos +tinham +tive +teve +tivemos +tiveram +tivera +tivéramos +tenha +tenhamos +tenham +tivesse +tivéssemos +tivessem +tiver +tivermos +tiverem +terei +terá +teremos +terão +teria +teríamos +teriam diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ro.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ro.txt new file mode 100644 index 000000000..4fdee90a5 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ro.txt @@ -0,0 +1,233 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +acea +aceasta +această +aceea +acei +aceia +acel +acela +acele +acelea +acest +acesta +aceste +acestea +aceşti +aceştia +acolo +acum +ai +aia +aibă +aici +al +ăla +ale +alea +ălea +altceva +altcineva +am +ar +are +aş +aşadar +asemenea +asta +ăsta +astăzi +astea +ăstea +ăştia +asupra +aţi +au +avea +avem +aveţi +azi +bine +bucur +bună +ca +că +căci +când +care +cărei +căror +cărui +cât +câte +câţi +către +câtva +ce +cel +ceva +chiar +cînd +cine +cineva +cît +cîte +cîţi +cîtva +contra +cu +cum +cumva +curând +curînd +da +dă +dacă +dar +datorită +de +deci +deja +deoarece +departe +deşi +din +dinaintea +dintr +dintre +drept +după +ea +ei +el +ele +eram +este +eşti +eu +face +fără +fi +fie +fiecare +fii +fim +fiţi +iar +ieri +îi +îl +îmi +împotriva +în +înainte +înaintea +încât +încît +încotro +între +întrucât +întrucît +îţi +la +lângă +le +li +lîngă +lor +lui +mă +mâine +mea +mei +mele +mereu +meu +mi +mine +mult +multă +mulţi +ne +nicăieri +nici +nimeni +nişte +noastră +noastre +noi +noştri +nostru +nu +ori +oricând +oricare +oricât +orice +oricînd +oricine +oricît +oricum +oriunde +până +pe +pentru +peste +pînă +poate +pot +prea +prima +primul +prin +printr +sa +să +săi +sale +sau +său +se +şi +sînt +sîntem +sînteţi +spre +sub +sunt +suntem +sunteţi +ta +tăi +tale +tău +te +ţi +ţie +tine +toată +toate +tot +toţi +totuşi +tu +un +una +unde +undeva +unei +unele +uneori +unor +vă +vi +voastră +voastre +voi +voştri +vostru +vouă +vreo +vreun diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ru.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ru.txt new file mode 100644 index 000000000..55271400c --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ru.txt @@ -0,0 +1,243 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/russian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | a russian stop word list. comments begin with vertical bar. each stop + | word is at the start of a line. + + | this is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + | letter `ё' is translated to `е'. + +и | and +в | in/into +во | alternative form +не | not +что | what/that +он | he +на | on/onto +я | i +с | from +со | alternative form +как | how +а | milder form of `no' (but) +то | conjunction and form of `that' +все | all +она | she +так | so, thus +его | him +но | but +да | yes/and +ты | thou +к | towards, by +у | around, chez +же | intensifier particle +вы | you +за | beyond, behind +бы | conditional/subj. particle +по | up to, along +только | only +ее | her +мне | to me +было | it was +вот | here is/are, particle +от | away from +меня | me +еще | still, yet, more +нет | no, there isnt/arent +о | about +из | out of +ему | to him +теперь | now +когда | when +даже | even +ну | so, well +вдруг | suddenly +ли | interrogative particle +если | if +уже | already, but homonym of `narrower' +или | or +ни | neither +быть | to be +был | he was +него | prepositional form of его +до | up to +вас | you accusative +нибудь | indef. suffix preceded by hyphen +опять | again +уж | already, but homonym of `adder' +вам | to you +сказал | he said +ведь | particle `after all' +там | there +потом | then +себя | oneself +ничего | nothing +ей | to her +может | usually with `быть' as `maybe' +они | they +тут | here +где | where +есть | there is/are +надо | got to, must +ней | prepositional form of ей +для | for +мы | we +тебя | thee +их | them, their +чем | than +была | she was +сам | self +чтоб | in order to +без | without +будто | as if +человек | man, person, one +чего | genitive form of `what' +раз | once +тоже | also +себе | to oneself +под | beneath +жизнь | life +будет | will be +ж | short form of intensifer particle `же' +тогда | then +кто | who +этот | this +говорил | was saying +того | genitive form of `that' +потому | for that reason +этого | genitive form of `this' +какой | which +совсем | altogether +ним | prepositional form of `его', `они' +здесь | here +этом | prepositional form of `этот' +один | one +почти | almost +мой | my +тем | instrumental/dative plural of `тот', `то' +чтобы | full form of `in order that' +нее | her (acc.) +кажется | it seems +сейчас | now +были | they were +куда | where to +зачем | why +сказать | to say +всех | all (acc., gen. preposn. plural) +никогда | never +сегодня | today +можно | possible, one can +при | by +наконец | finally +два | two +об | alternative form of `о', about +другой | another +хоть | even +после | after +над | above +больше | more +тот | that one (masc.) +через | across, in +эти | these +нас | us +про | about +всего | in all, only, of all +них | prepositional form of `они' (they) +какая | which, feminine +много | lots +разве | interrogative particle +сказала | she said +три | three +эту | this, acc. fem. sing. +моя | my, feminine +впрочем | moreover, besides +хорошо | good +свою | ones own, acc. fem. sing. +этой | oblique form of `эта', fem. `this' +перед | in front of +иногда | sometimes +лучше | better +чуть | a little +том | preposn. form of `that one' +нельзя | one must not +такой | such a one +им | to them +более | more +всегда | always +конечно | of course +всю | acc. fem. sing of `all' +между | between + + + | b: some paradigms + | + | personal pronouns + | + | я меня мне мной [мною] + | ты тебя тебе тобой [тобою] + | он его ему им [него, нему, ним] + | она ее эи ею [нее, нэи, нею] + | оно его ему им [него, нему, ним] + | + | мы нас нам нами + | вы вас вам вами + | они их им ими [них, ним, ними] + | + | себя себе собой [собою] + | + | demonstrative pronouns: этот (this), тот (that) + | + | этот эта это эти + | этого эты это эти + | этого этой этого этих + | этому этой этому этим + | этим этой этим [этою] этими + | этом этой этом этих + | + | тот та то те + | того ту то те + | того той того тех + | тому той тому тем + | тем той тем [тою] теми + | том той том тех + | + | determinative pronouns + | + | (a) весь (all) + | + | весь вся все все + | всего всю все все + | всего всей всего всех + | всему всей всему всем + | всем всей всем [всею] всеми + | всем всей всем всех + | + | (b) сам (himself etc) + | + | сам сама само сами + | самого саму само самих + | самого самой самого самих + | самому самой самому самим + | самим самой самим [самою] самими + | самом самой самом самих + | + | stems of verbs `to be', `to have', `to do' and modal + | + | быть бы буд быв есть суть + | име + | дел + | мог мож мочь + | уме + | хоч хот + | долж + | можн + | нужн + | нельзя + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_sv.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_sv.txt new file mode 100644 index 000000000..096f87f67 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_sv.txt @@ -0,0 +1,133 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/swedish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Swedish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + | Swedish stop words occasionally exhibit homonym clashes. For example + | så = so, but also seed. These are indicated clearly below. + +och | and +det | it, this/that +att | to (with infinitive) +i | in, at +en | a +jag | I +hon | she +som | who, that +han | he +på | on +den | it, this/that +med | with +var | where, each +sig | him(self) etc +för | for +så | so (also: seed) +till | to +är | is +men | but +ett | a +om | if; around, about +hade | had +de | they, these/those +av | of +icke | not, no +mig | me +du | you +henne | her +då | then, when +sin | his +nu | now +har | have +inte | inte någon = no one +hans | his +honom | him +skulle | 'sake' +hennes | her +där | there +min | my +man | one (pronoun) +ej | nor +vid | at, by, on (also: vast) +kunde | could +något | some etc +från | from, off +ut | out +när | when +efter | after, behind +upp | up +vi | we +dem | them +vara | be +vad | what +över | over +än | than +dig | you +kan | can +sina | his +här | here +ha | have +mot | towards +alla | all +under | under (also: wonder) +någon | some etc +eller | or (else) +allt | all +mycket | much +sedan | since +ju | why +denna | this/that +själv | myself, yourself etc +detta | this/that +åt | to +utan | without +varit | was +hur | how +ingen | no +mitt | my +ni | you +bli | to be, become +blev | from bli +oss | us +din | thy +dessa | these/those +några | some etc +deras | their +blir | from bli +mina | my +samma | (the) same +vilken | who, that +er | you, your +sådan | such a +vår | our +blivit | from bli +dess | its +inom | within +mellan | between +sådant | such a +varför | why +varje | each +vilka | who, that +ditt | thy +vem | who +vilket | who, that +sitta | his +sådana | such a +vart | each +dina | thy +vars | whose +vårt | our +våra | our +ert | your +era | your +vilkas | whose + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_th.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_th.txt new file mode 100644 index 000000000..07f0fabe6 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_th.txt @@ -0,0 +1,119 @@ +# Thai stopwords from: +# "Opinion Detection in Thai Political News Columns +# Based on Subjectivity Analysis" +# Khampol Sukhum, Supot Nitsuwat, and Choochart Haruechaiyasak +ไว้ +ไม่ +ไป +ได้ +ให้ +ใน +โดย +แห่ง +แล้ว +และ +แรก +แบบ +แต่ +เอง +เห็น +เลย +เริ่ม +เรา +เมื่อ +เพื่อ +เพราะ +เป็นการ +เป็น +เปิดเผย +เปิด +เนื่องจาก +เดียวกัน +เดียว +เช่น +เฉพาะ +เคย +เข้า +เขา +อีก +อาจ +อะไร +ออก +อย่าง +อยู่ +อยาก +หาก +หลาย +หลังจาก +หลัง +หรือ +หนึ่ง +ส่วน +ส่ง +สุด +สําหรับ +ว่า +วัน +ลง +ร่วม +ราย +รับ +ระหว่าง +รวม +ยัง +มี +มาก +มา +พร้อม +พบ +ผ่าน +ผล +บาง +น่า +นี้ +นํา +นั้น +นัก +นอกจาก +ทุก +ที่สุด +ที่ +ทําให้ +ทํา +ทาง +ทั้งนี้ +ทั้ง +ถ้า +ถูก +ถึง +ต้อง +ต่างๆ +ต่าง +ต่อ +ตาม +ตั้งแต่ +ตั้ง +ด้าน +ด้วย +ดัง +ซึ่ง +ช่วง +จึง +จาก +จัด +จะ +คือ +ความ +ครั้ง +คง +ขึ้น +ของ +ขอ +ขณะ +ก่อน +ก็ +การ +กับ +กัน +กว่า +กล่าว diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_tr.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_tr.txt new file mode 100644 index 000000000..84d9408d4 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_tr.txt @@ -0,0 +1,212 @@ +# Turkish stopwords from LUCENE-559 +# merged with the list from "Information Retrieval on Turkish Texts" +# (http://www.users.muohio.edu/canf/papers/JASIST2008offPrint.pdf) +acaba +altmış +altı +ama +ancak +arada +aslında +ayrıca +bana +bazı +belki +ben +benden +beni +benim +beri +beş +bile +bin +bir +birçok +biri +birkaç +birkez +birşey +birşeyi +biz +bize +bizden +bizi +bizim +böyle +böylece +bu +buna +bunda +bundan +bunlar +bunları +bunların +bunu +bunun +burada +çok +çünkü +da +daha +dahi +de +defa +değil +diğer +diye +doksan +dokuz +dolayı +dolayısıyla +dört +edecek +eden +ederek +edilecek +ediliyor +edilmesi +ediyor +eğer +elli +en +etmesi +etti +ettiği +ettiğini +gibi +göre +halen +hangi +hatta +hem +henüz +hep +hepsi +her +herhangi +herkesin +hiç +hiçbir +için +iki +ile +ilgili +ise +işte +itibaren +itibariyle +kadar +karşın +katrilyon +kendi +kendilerine +kendini +kendisi +kendisine +kendisini +kez +ki +kim +kimden +kime +kimi +kimse +kırk +milyar +milyon +mu +mü +mı +nasıl +ne +neden +nedenle +nerde +nerede +nereye +niye +niçin +o +olan +olarak +oldu +olduğu +olduğunu +olduklarını +olmadı +olmadığı +olmak +olması +olmayan +olmaz +olsa +olsun +olup +olur +olursa +oluyor +on +ona +ondan +onlar +onlardan +onları +onların +onu +onun +otuz +oysa +öyle +pek +rağmen +sadece +sanki +sekiz +seksen +sen +senden +seni +senin +siz +sizden +sizi +sizin +şey +şeyden +şeyi +şeyler +şöyle +şu +şuna +şunda +şundan +şunları +şunu +tarafından +trilyon +tüm +üç +üzere +var +vardı +ve +veya +ya +yani +yapacak +yapılan +yapılması +yapıyor +yapmak +yaptı +yaptığı +yaptığını +yaptıkları +yedi +yerine +yetmiş +yine +yirmi +yoksa +yüz +zaten diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/userdict_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/userdict_ja.txt new file mode 100644 index 000000000..6f0368e4d --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/userdict_ja.txt @@ -0,0 +1,29 @@ +# +# This is a sample user dictionary for Kuromoji (JapaneseTokenizer) +# +# Add entries to this file in order to override the statistical model in terms +# of segmentation, readings and part-of-speech tags. Notice that entries do +# not have weights since they are always used when found. This is by-design +# in order to maximize ease-of-use. +# +# Entries are defined using the following CSV format: +# , ... , ... , +# +# Notice that a single half-width space separates tokens and readings, and +# that the number tokens and readings must match exactly. +# +# Also notice that multiple entries with the same is undefined. +# +# Whitespace only lines are ignored. Comments are not allowed on entry lines. +# + +# Custom segmentation for kanji compounds +日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞 +関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,カスタム名詞 + +# Custom segmentation for compound katakana +トートバッグ,トート バッグ,トート バッグ,かずカナ名詞 +ショルダーバッグ,ショルダー バッグ,ショルダー バッグ,かずカナ名詞 + +# Custom reading for former sumo wrestler +朝青龍,朝青龍,アサショウリュウ,カスタム人名 diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/protwords.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/protwords.txt new file mode 100644 index 000000000..1dfc0abec --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/schema-rerank.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/schema-rerank.xml new file mode 100644 index 000000000..6cd4159df --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/schema-rerank.xml @@ -0,0 +1,410 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/schema.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/schema.xml new file mode 100644 index 000000000..024c6ebb2 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/schema.xml @@ -0,0 +1,766 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.snippet.randomindexconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.snippet.randomindexconfig.xml new file mode 100644 index 000000000..7514aa478 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.snippet.randomindexconfig.xml @@ -0,0 +1,47 @@ + + + + + + + + + ${useCompoundFile:false} + + ${solr.tests.maxBufferedDocs} + ${solr.tests.maxIndexingThreads} + ${solr.tests.ramBufferSizeMB} + + + + 1000 + 10000 + + + ${solr.tests.lockType:single} + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml new file mode 100644 index 000000000..ad40a86d2 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml @@ -0,0 +1,570 @@ + + + + + + + + + + + + ${solr.data.dir:} + + + + 1000000 + 2000000 + 3000000 + 4000000 + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + true + + + + + + 10 + + + + + + + + + + + + + + + + + + + + + + + http://127.0.0.1:8983/solr/${solr.core.name} + 00:00:20 + /Users/agazzarini/workspaces/alfresco/spike-custom-replication-handler/src/test/resources/contentstore + + + + + + + + true + + + + + true + + + + + + dismax + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + + + fingerprint + + + + + + afts + false + false + 5 + 2 + 5 + true + true + 5 + 3 + mltext@m___t@{http://www.alfresco.org/model/content/1.0}title + id + content@s___t@{http://www.alfresco.org/model/content/1.0}content + true + false + + + setLocale + rewriteFacetParameters + query + facet + facet_module + mlt + highlight + stats + debug + clearLocale + rewriteFacetCounts + spellcheck + spellcheckbackcompat + setProcessedDenies + + + + + + + + explicit + 10 + suggest + + + + + setLocale + query + facet + mlt + highlight + stats + debug + clearLocale + + + + + + + + cmis + + + query + facet + mlt + highlight + stats + debug + + + + + + + + + termsComp + + + + + + + + + + + + + + + tvComponent + + + + + + + + + + + + 100 + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + max-age=30, public + + + + + + + explicit + true + + + + + solr + solrconfig.xml schema.xml admin-extra.html + + + + + + + + + + + + + + + + + + conf/mime_types.csv + + + + 1 + 10 + + + + + + + + text_shingle + + + + + + default + suggest + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + wordbreak + suggest + solr.WordBreakSolrSpellChecker + true + true + 10 + 5 + + + + + + + + + + + + + + + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrcore.properties b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrcore.properties new file mode 100644 index 000000000..a46ace7f4 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrcore.properties @@ -0,0 +1,154 @@ +# +# solrcore.properties - used in solrconfig.xml +# + +enable.alfresco.tracking=false + +# +#These are replaced by the admin handler +# +#data.dir.root=DATA_DIR +#data.dir.store=workspace/SpacesStore +#alfresco.stores=workspace://SpacesStore + +# +# Properties loaded during alfresco tracking +# + +alfresco.host=localhost +alfresco.port=8080 +alfresco.port.ssl=8443 +alfresco.baseUrl=/alfresco +alfresco.cron=0/2 * * * * ? * + +#alfresco.index.transformContent=false +#alfresco.ignore.datatype.1=d:content +alfresco.lag=1000 +alfresco.hole.retention=3600000 +# alfresco.hole.check.after is not used yet +# It will reduce the hole checking load +alfresco.hole.check.after=300000 +alfresco.batch.count=1000 + +# encryption + +# none, https +alfresco.secureComms=none + +# ssl +alfresco.encryption.ssl.keystore.type=JCEKS +alfresco.encryption.ssl.keystore.provider= +alfresco.encryption.ssl.keystore.location=ssl.repo.client.keystore +alfresco.encryption.ssl.keystore.passwordFileLocation=ssl-keystore-passwords.properties +alfresco.encryption.ssl.truststore.type=JCEKS +alfresco.encryption.ssl.truststore.provider= +alfresco.encryption.ssl.truststore.location=ssl.repo.client.truststore +alfresco.encryption.ssl.truststore.passwordFileLocation=ssl-truststore-passwords.properties + +# Tracking + +alfresco.corePoolSize=1 +alfresco.maximumPoolSize=-1 +alfresco.keepAliveTime=120 +alfresco.threadPriority=5 +alfresco.threadDaemon=true +alfresco.workQueueSize=-1 + +# HTTP Client + +alfresco.maxTotalConnections=200 +alfresco.maxHostConnections=200 +alfresco.socketTimeout=360000 + +# SOLR caching + +solr.filterCache.size=256 +solr.filterCache.initialSize=128 +solr.queryResultCache.size=1024 +solr.queryResultCache.initialSize=1024 +solr.documentCache.size=1024 +solr.documentCache.initialSize=1024 +solr.queryResultMaxDocsCached=2048 + +solr.authorityCache.size=128 +solr.authorityCache.initialSize=64 +solr.pathCache.size=256 +solr.pathCache.initialSize=128 + +solr.ownerCache.size=128 +solr.ownerCache.initialSize=64 + +solr.readerCache.size=128 +solr.readerCache.initialSize=64 + +solr.deniedCache.size=128 +solr.deniedCache.initialSize=64 + +# SOLR + +solr.maxBooleanClauses=10000 + +# Batch fetch + +alfresco.transactionDocsBatchSize=100 +alfresco.nodeBatchSize=10 +alfresco.changeSetAclsBatchSize=100 +alfresco.aclBatchSize=10 +alfresco.contentReadBatchSize=4000 +alfresco.contentUpdateBatchSize=1000 + +# Warming + +solr.filterCache.autowarmCount=32 +solr.authorityCache.autowarmCount=4 +solr.pathCache.autowarmCount=32 +solr.deniedCache.autowarmCount=0 +solr.readerCache.autowarmCount=0 +solr.ownerCache.autowarmCount=0 +solr.queryResultCache.autowarmCount=4 +solr.documentCache.autowarmCount=512 + +solr.queryResultWindowSize=512 + + +# +# TODO +# +# cross language support +# locale expansion +# logging check report .... +# +# + +alfresco.commitInterval=1000 +alfresco.newSearcherInterval=2000 + +alfresco.doPermissionChecks=true + + +# +# Metadata pulling control +# +alfresco.metadata.skipDescendantDocsForSpecificTypes=false +alfresco.metadata.ignore.datatype.0=cm:person +alfresco.metadata.ignore.datatype.1=app:configurations +alfresco.metadata.skipDescendantDocsForSpecificAspects=false +#alfresco.metadata.ignore.aspect.0= + + +# +# Suggestions +# +solr.suggester.enabled=false +# -1 to disable suggester build throttling +solr.suggester.minSecsBetweenBuilds=3600 + +# +# Limit the maximum text size of transformed content sent to the index - in bytes +# +alfresco.contentStreamLimit=10000000 + +#Sharding default values +shard.instance=1 +shard.count=0 +shard.method=DB_ID diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/spellings.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/spellings.txt new file mode 100644 index 000000000..d7ede6f56 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/spellings.txt @@ -0,0 +1,2 @@ +pizza +history \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/stopwords.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/stopwords.txt new file mode 100644 index 000000000..6f46b1bf4 --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/stopwords.txt @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/synonyms.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/synonyms.txt new file mode 100644 index 000000000..74cea68ad --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/synonyms.txt @@ -0,0 +1,35 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaa => aaaa +bbb => bbbb1 bbbb2 +ccc => cccc1,cccc2 +a\=>a => b\=>b +a\,a => b\,b +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + +# Test synonyms +quick,fast,rapid,speedy +brown fox jumped,leaping reynard,springer +lazy,bone idle \ No newline at end of file From 3d63579d9b20a9ec4d84832bc7a9d4fd9f7d55a9 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Wed, 16 Oct 2019 23:16:02 +0200 Subject: [PATCH 30/76] [contentStoreReplication] added test for contentstore replication --- .../handler/contentStoreReplicationTest.java | 252 ++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java new file mode 100644 index 000000000..6f6172afe --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.handler; + +import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.client.Acl; +import org.alfresco.solr.client.AclChangeSet; +import org.alfresco.solr.client.AclReaders; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.Transaction; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Properties; + +import static java.util.Collections.singletonList; +import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; +import static org.alfresco.solr.AlfrescoSolrUtils.getNode; +import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; +import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; +import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; +import static org.alfresco.solr.utils.AlfrescoFileUtils.areDirectoryEquals; +import static org.carrot2.shaded.guava.common.collect.ImmutableList.of; + +/** + * @author Elia Porciani + */ +public class contentStoreReplicationTest extends AbstractAlfrescoDistributedTest { + + protected static JettySolrRunner master; + protected static JettySolrRunner slave; + protected static SolrClient masterClient; + protected static SolrClient slaveClient; + + protected final static int MASTER_PORT = 2345; + + protected static Path masterSolrHome; + protected static Path slaveSolrHome; + + protected static Path masterContentStore; + protected static Path slaveContentStore; + + private static Acl acl; + + private static final int MILLIS_TIMOUT = 80000; + + + @BeforeClass + public static void createMasterSlaveEnv() throws Exception + { + Properties properties = new Properties(); + + clientShards = new ArrayList<>(); + solrShards = new ArrayList<>(); + solrCollectionNameToStandaloneClient = new HashMap<>(); + jettyContainers = new HashMap<>(); + + String coreName = "master"; + boolean basicAuth = properties != null ? Boolean.parseBoolean(properties.getProperty("BasicAuth", "false")) : false; + + String masterKey = "master/solrHome"; + String slaveKey = "slave/solrHome"; + + master = createJetty(masterKey, basicAuth, MASTER_PORT); + addCoreToJetty(masterKey, coreName, coreName, null); + startJetty(master); + + String slaveCoreName = "slave"; + slave = createJetty(slaveKey, basicAuth); + addCoreToJetty(slaveKey, slaveCoreName, slaveCoreName, null); + startJetty(slave); + + String masterStr = buildUrl(master.getLocalPort()) + "/" + coreName; + String slaveStr = buildUrl(slave.getLocalPort()) + "/" + slaveCoreName; + + masterClient = createNewSolrClient(masterStr); + slaveClient = createNewSolrClient(slaveStr); + + masterSolrHome = testDir.toPath().resolve(masterKey); + slaveSolrHome = testDir.toPath().resolve(slaveKey); + masterContentStore = testDir.toPath().resolve("master/contentstore"); + slaveContentStore = testDir.toPath().resolve("slave/contentstore"); + + AclChangeSet aclChangeSet = getAclChangeSet(1); + + acl = getAcl(aclChangeSet); + AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + + indexAclChangeSet(aclChangeSet, + of(acl), + of(aclReaders)); + } + + @AfterClass + public static void cleanupMasterSlave() throws Exception { + master.stop(); + slave.stop(); + FileUtils.forceDelete(new File(masterSolrHome.getParent().toUri())); + FileUtils.forceDelete(new File(slaveSolrHome.getParent().toUri())); + } + + + + @Test + public void contentStoreReplicationTest() throws Exception { + // ADD 250 nodes and check they are replicated + int numNodes = 250; + Transaction bigTxn = getTransaction(0, numNodes); + List nodes = new ArrayList<>(); + List nodeMetaDatas = new ArrayList<>(); + for(int i = 0; i updateNodes = new ArrayList<>(); + List updateNodeMetaDatas = new ArrayList<>(); + for(int i = numNodes; i < totalNodes; i++) { + Node node = getNode(i, updateTx, acl, Node.SolrApiNodeStatus.UPDATED); + updateNodes.add(node); + NodeMetaData nodeMetaData = getNodeMetaData(node, updateTx, acl, "mike", null, false); + node.setNodeRef(nodeMetaData.getNodeRef().toString()); + updateNodeMetaDatas.add(nodeMetaData); + } + + indexTransaction(updateTx, updateNodes, updateNodeMetaDatas); + + waitForDocCountCore(masterClient, + luceneToSolrQuery(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world"))), + numNodes + numUpdates, MILLIS_TIMOUT, System.currentTimeMillis()); + + filesInMasterContentStore = Files.walk(Paths.get(masterContentStore.toUri().resolve("_DEFAULT_"))) + .filter(Files::isRegularFile) + .count(); + + Assert.assertEquals( "master contentStore should have " + totalNodes + "files", totalNodes, filesInMasterContentStore); + assertTrue("slave content store is not in sync after timeout", waitForContentStoreSync(MILLIS_TIMOUT)); + + + // DELETES 30 nodes + int numDeletes = 30; + Transaction deleteTx = getTransaction(numDeletes, 0); + + totalNodes = numNodes + numUpdates - numDeletes; + List deleteNodes = new ArrayList<>(); + List deleteNodeMetaDatas = new ArrayList<>(); + + for(int i = 0; i Date: Wed, 16 Oct 2019 23:22:29 +0200 Subject: [PATCH 31/76] [contentStoreReplication] fix slave configuration --- .../test/resources/test-files/slave/conf/solrconfig.xml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml index ad40a86d2..9aefc8667 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml @@ -188,13 +188,10 @@ - - - http://127.0.0.1:8983/solr/${solr.core.name} - 00:00:20 - /Users/agazzarini/workspaces/alfresco/spike-custom-replication-handler/src/test/resources/contentstore + http://127.0.0.1:2345/solr/master + 00:00:02 From 8e14d5a9ad8e3fbfac2b485a98477c5721908c32 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 17 Oct 2019 17:18:19 +0200 Subject: [PATCH 32/76] [contentStoreReplication] added documentation --- .../org/alfresco/solr/utils/AlfrescoFileUtils.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/AlfrescoFileUtils.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/AlfrescoFileUtils.java index 3ae2221d4..dcbf963cf 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/AlfrescoFileUtils.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/AlfrescoFileUtils.java @@ -33,6 +33,16 @@ import java.util.stream.Collectors; * @author Elia Porciani */ public class AlfrescoFileUtils { + + /** + * Check if two directories contains the same files + * + * @param dir + * @param dir2 + * @param extensions Limits the search to the extensions provided + * @param recursive Check recursively in all subdirs + * @return + */ public static boolean areDirectoryEquals(Path dir, Path dir2, String[] extensions, boolean recursive) { Map filesDir1 = FileUtils.listFiles(new File(dir.toUri()), extensions, recursive) From 3823a5bef2aa90b0eeb5e47ec5e9821973fbbff5 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 17 Oct 2019 17:20:21 +0200 Subject: [PATCH 33/76] [contentStoreReplication] Created new directory for test execution different from test-files Removed test execution files after test --- .../solr/AbstractAlfrescoSolrTests.java | 41 +++++++++++++++---- .../java/org/alfresco/solr/SolrTestFiles.java | 10 +++-- .../handler/contentStoreReplicationTest.java | 4 +- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java index a7647a399..956fb829a 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java @@ -27,7 +27,6 @@ import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.client.SOLRAPIQueueClient; import org.alfresco.solr.client.Transaction; import org.alfresco.solr.tracker.Tracker; -import org.alfresco.util.SearchLanguageConversion; import org.apache.chemistry.opencmis.commons.impl.json.JSONArray; import org.apache.chemistry.opencmis.commons.impl.json.JSONObject; import org.apache.chemistry.opencmis.commons.impl.json.JSONValue; @@ -67,6 +66,7 @@ import org.xml.sax.SAXException; import javax.servlet.http.HttpServletRequest; import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPathExpressionException; +import java.io.File; import java.io.IOException; import java.math.BigInteger; import java.nio.file.Paths; @@ -119,6 +119,26 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres protected static NodeRef TEST_NODEREF; protected static Date FTS_TEST_DATE; + + + protected static void copyTestFiles() throws IOException { + + //Add solr home conf folder with alfresco based configuration. + + File testExecutionFolder = Paths.get(TEST_EXECUTION_SOLRHOME).toFile(); + + for (String s : List.of("/conf", "/alfrescoModels", "/templates", "/collection1")) { + FileUtils.copyDirectory(Paths.get(TEST_FILES_LOCATION + s).toFile(), Paths.get(testExecutionFolder + s).toFile()); + } + } + + @AfterClass + public static void deleteTestDirectory() throws IOException { + FileUtils.forceDelete(Paths.get(TEST_EXECUTION_FOLDER).toFile()); + } + + + /* Bunch of methods that wrap testHarness object usage. * TestHarness is a class for internal solr test and should not be use outside Solr. * Unfortunately this class is used in too many test and a complete refactor would have a huge impact @@ -218,9 +238,9 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres @Deprecated public static void initAlfrescoCore(String schema) throws Exception { + LOG.info("##################################### init Alfresco core ##############"); LOG.info("####initCore"); - System.setProperty("solr.solr.home", TEST_FILES_LOCATION); System.setProperty("solr.directoryFactory","solr.RAMDirectoryFactory"); System.setProperty("solr.tests.maxBufferedDocs", "1000"); System.setProperty("solr.tests.maxIndexingThreads", "10"); @@ -231,18 +251,23 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres System.setProperty("alfresco.test", "true"); System.setProperty("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler"); System.setProperty("solr.tests.mergePolicy", "org.apache.lucene.index.TieredMergePolicy"); + + copyTestFiles(); if (CORE_NOT_YET_CREATED) { createAlfrescoCore(schema); } LOG.info("####initCore end"); + + admin = of(h).map(TestHarness::getCoreContainer) .map(CoreContainer::getMultiCoreHandler) .map(AlfrescoCoreAdminHandler.class::cast) .orElseThrow(RuntimeException::new); } + /** * @deprecated as testHarness is used */ @@ -267,7 +292,7 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres Paths.get(TEST_SOLR_CONF + schema).toFile()); } - SolrResourceLoader resourceLoader = new SolrResourceLoader(Paths.get(TEST_FILES_LOCATION), null, properties); + SolrResourceLoader resourceLoader = new SolrResourceLoader(Paths.get(TEST_EXECUTION_SOLRHOME), null, properties); TestCoresLocator locator = new TestCoresLocator(SolrTestCaseJ4.DEFAULT_TEST_CORENAME, "data", "solrconfig.xml", @@ -309,7 +334,7 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres /** * Validates an update XML String is successful */ - public void assertU(String update) + public void assertU(String update) { assertU(null, update); } @@ -325,7 +350,7 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres /** * Validates an update XML String failed */ - public void assertFailedU(String update) + public void assertFailedU(String update) { assertFailedU(null, update); } @@ -434,6 +459,7 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres } catch (Exception e2) { + e2.printStackTrace(); throw new RuntimeException("Exception during query", e2); } } @@ -498,7 +524,7 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres throws Exception { Date date = new Date(); - long timeout = (long)date.getTime() + waitMillis; + long timeout = date.getTime() + waitMillis; RefCounted ref = null; int totalHits = 0; @@ -738,6 +764,7 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres } catch(Exception exception) { + exception.printStackTrace(); throw new RuntimeException(exception); } finally @@ -785,7 +812,7 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres { public SolrServletRequest(SolrCore core, HttpServletRequest req) { - super(core, new MultiMapSolrParams(Collections. emptyMap())); + super(core, new MultiMapSolrParams(Collections.emptyMap())); } } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestFiles.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestFiles.java index cf020be7b..86a505ad1 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestFiles.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestFiles.java @@ -24,8 +24,10 @@ package org.alfresco.solr; */ public interface SolrTestFiles { - public final String TEST_FILES_LOCATION = "target/test-classes/test-files"; - public final String TEST_SOLR_COLLECTION = TEST_FILES_LOCATION + "/collection1"; - public final String TEST_SOLR_CONF = TEST_SOLR_COLLECTION + "/conf/"; - public final String TEMPLATE_CONF = TEST_FILES_LOCATION + "/templates/%s/conf/"; + String TEST_FILES_LOCATION = "target/test-classes/test-files"; + String TEST_EXECUTION_FOLDER = "target/test-execution-folder"; + String TEST_EXECUTION_SOLRHOME = TEST_EXECUTION_FOLDER + "/solrhome"; + String TEST_SOLR_COLLECTION = TEST_EXECUTION_SOLRHOME + "/collection1"; + String TEST_SOLR_CONF = TEST_SOLR_COLLECTION + "/conf/"; + String TEMPLATE_CONF = TEST_EXECUTION_SOLRHOME + "/templates/%s/conf/"; } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java index 6f6172afe..7f06373af 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java @@ -58,6 +58,8 @@ import static org.carrot2.shaded.guava.common.collect.ImmutableList.of; /** * @author Elia Porciani + * + * This test check if the synchronization of contentstore between master and slave is done correctly. */ public class contentStoreReplicationTest extends AbstractAlfrescoDistributedTest { @@ -125,6 +127,7 @@ public class contentStoreReplicationTest extends AbstractAlfrescoDistributedTest of(aclReaders)); } + @AfterClass public static void cleanupMasterSlave() throws Exception { master.stop(); @@ -134,7 +137,6 @@ public class contentStoreReplicationTest extends AbstractAlfrescoDistributedTest } - @Test public void contentStoreReplicationTest() throws Exception { // ADD 250 nodes and check they are replicated From 3afd7a5cb4ec4bcecdedc83e2aa796b547de677e Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 18 Oct 2019 17:04:54 +0200 Subject: [PATCH 34/76] [contentStoreReplication] Created different folders for each test extending AbstractAlfrescoSolrTests in order to avoid problems with contentStore lucene index. Code refactoring. --- .../solr/AbstractAlfrescoSolrTests.java | 33 +++++++++++-------- .../java/org/alfresco/solr/SolrTestFiles.java | 4 --- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java index 956fb829a..80b2cb7b0 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java @@ -33,6 +33,7 @@ import org.apache.chemistry.opencmis.commons.impl.json.JSONValue; import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.Time; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.solr.SolrTestCaseJ4; @@ -97,6 +98,12 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres private static Log LOG = LogFactory.getLog(AbstractAlfrescoSolrTests.class); private static boolean CORE_NOT_YET_CREATED = true; + + private static String testExecutionSolrHome; + private static String testSolrCollection; + private static String testSolrConf; + private static String templateConf; + /** * Harness initialized by initTestHarness. *

@@ -125,19 +132,13 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres //Add solr home conf folder with alfresco based configuration. - File testExecutionFolder = Paths.get(TEST_EXECUTION_SOLRHOME).toFile(); + File testExecutionFolder = Paths.get(testExecutionSolrHome).toFile(); for (String s : List.of("/conf", "/alfrescoModels", "/templates", "/collection1")) { FileUtils.copyDirectory(Paths.get(TEST_FILES_LOCATION + s).toFile(), Paths.get(testExecutionFolder + s).toFile()); } } - @AfterClass - public static void deleteTestDirectory() throws IOException { - FileUtils.forceDelete(Paths.get(TEST_EXECUTION_FOLDER).toFile()); - } - - /* Bunch of methods that wrap testHarness object usage. * TestHarness is a class for internal solr test and should not be use outside Solr. @@ -252,9 +253,15 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres System.setProperty("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler"); System.setProperty("solr.tests.mergePolicy", "org.apache.lucene.index.TieredMergePolicy"); - copyTestFiles(); if (CORE_NOT_YET_CREATED) { + + testExecutionSolrHome = TEST_EXECUTION_FOLDER + "/" + Time.now() + "/solrhome"; + testSolrCollection = testExecutionSolrHome + "/collection1"; + testSolrConf = testSolrCollection + "/conf/"; + templateConf = testExecutionSolrHome + "/templates/%s/conf/"; + + copyTestFiles(); createAlfrescoCore(schema); } LOG.info("####initCore end"); @@ -288,11 +295,11 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres { String templateName = System.getProperty("templateName", "rerank"); FileUtils.copyFile( - Paths.get(String.format(TEMPLATE_CONF, templateName) + schema).toFile(), - Paths.get(TEST_SOLR_CONF + schema).toFile()); + Paths.get(String.format(templateConf, templateName) + schema).toFile(), + Paths.get(testSolrConf + schema).toFile()); } - SolrResourceLoader resourceLoader = new SolrResourceLoader(Paths.get(TEST_EXECUTION_SOLRHOME), null, properties); + SolrResourceLoader resourceLoader = new SolrResourceLoader(Paths.get(testExecutionSolrHome), null, properties); TestCoresLocator locator = new TestCoresLocator(SolrTestCaseJ4.DEFAULT_TEST_CORENAME, "data", "solrconfig.xml", @@ -317,9 +324,9 @@ public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, Alfres } @AfterClass() - public static void tearDown() - { + public static void tearDown() throws IOException { h.close(); + FileUtils.forceDelete(Paths.get(TEST_EXECUTION_FOLDER).toFile()); CORE_NOT_YET_CREATED = true; } /** diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestFiles.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestFiles.java index 86a505ad1..287701511 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestFiles.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestFiles.java @@ -26,8 +26,4 @@ public interface SolrTestFiles { String TEST_FILES_LOCATION = "target/test-classes/test-files"; String TEST_EXECUTION_FOLDER = "target/test-execution-folder"; - String TEST_EXECUTION_SOLRHOME = TEST_EXECUTION_FOLDER + "/solrhome"; - String TEST_SOLR_COLLECTION = TEST_EXECUTION_SOLRHOME + "/collection1"; - String TEST_SOLR_CONF = TEST_SOLR_COLLECTION + "/conf/"; - String TEMPLATE_CONF = TEST_EXECUTION_SOLRHOME + "/templates/%s/conf/"; } From d385a13e458e04b7ed0c53881bbd18ae8741ffd8 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 18 Oct 2019 17:06:23 +0200 Subject: [PATCH 35/76] [contentStoreReplication] Handled the case in which corecontainer is null (this can happen in unit tests) --- .../java/org/alfresco/solr/AlfrescoCoreAdminHandler.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index c748cc3fa..92cec28fe 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -138,7 +138,11 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler trackerRegistry = new TrackerRegistry(); informationServers = new ConcurrentHashMap<>(); this.scheduler = new SolrTrackerScheduler(this); - this.contentStore = new SolrContentStore(coreContainer.getSolrHome()); + + if (coreContainer != null) + { + this.contentStore = new SolrContentStore(coreContainer.getSolrHome()); + } String createDefaultCores = ConfigUtil.locateProperty(ALFRESCO_DEFAULTS, ""); int numShards = Integer.valueOf(ConfigUtil.locateProperty(NUM_SHARDS, "1")); From d41dec912ca5ff0aab72bccd1cf01958a467b824 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 22 Oct 2019 16:57:01 +0100 Subject: [PATCH 36/76] [contentStoreReplication] fix test: set master address on slave/solrconfig.xml dynamically --- .../handler/contentStoreReplicationTest.java | 21 +++++++++++++++---- .../test-files/slave/conf/solrconfig.xml | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java index 7f06373af..fd4c1e24f 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java @@ -18,6 +18,7 @@ */ package org.alfresco.solr.handler; +import java.io.IOException; import org.alfresco.solr.AbstractAlfrescoDistributedTest; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; @@ -68,8 +69,6 @@ public class contentStoreReplicationTest extends AbstractAlfrescoDistributedTest protected static SolrClient masterClient; protected static SolrClient slaveClient; - protected final static int MASTER_PORT = 2345; - protected static Path masterSolrHome; protected static Path slaveSolrHome; @@ -80,7 +79,6 @@ public class contentStoreReplicationTest extends AbstractAlfrescoDistributedTest private static final int MILLIS_TIMOUT = 80000; - @BeforeClass public static void createMasterSlaveEnv() throws Exception { @@ -97,13 +95,16 @@ public class contentStoreReplicationTest extends AbstractAlfrescoDistributedTest String masterKey = "master/solrHome"; String slaveKey = "slave/solrHome"; - master = createJetty(masterKey, basicAuth, MASTER_PORT); + master = createJetty(masterKey, basicAuth); addCoreToJetty(masterKey, coreName, coreName, null); startJetty(master); + String slaveCoreName = "slave"; slave = createJetty(slaveKey, basicAuth); addCoreToJetty(slaveKey, slaveCoreName, slaveCoreName, null); + setMasterUrl(slaveKey, slaveCoreName, master.getBaseUrl().toString() + "/master"); + startJetty(slave); String masterStr = buildUrl(master.getLocalPort()) + "/" + coreName; @@ -251,4 +252,16 @@ public class contentStoreReplicationTest extends AbstractAlfrescoDistributedTest return false; } + private static void setMasterUrl(String jettyKey, String coreName, String masterUrl) throws IOException { + Path jettySolrHome = testDir.toPath().resolve(jettyKey); + Path coreHome = jettySolrHome.resolve(coreName); + Path confDir = coreHome.resolve("conf"); + + Path solrConfigPath = confDir.resolve("solrconfig.xml"); + + String content = new String(Files.readAllBytes(solrConfigPath)); + content = content.replaceAll("\\{masterURL\\}", masterUrl); + Files.write(solrConfigPath, content.getBytes()); + } + } diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml index 9aefc8667..790e1af2c 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml @@ -190,7 +190,7 @@ - http://127.0.0.1:2345/solr/master + {masterURL} 00:00:02 From ebc2b4ebdd25ce467074aff95e4e67fb38fa3c20 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Sun, 27 Oct 2019 18:35:41 +0100 Subject: [PATCH 37/76] [contentStoreReplication] removed solrCoreLoadRegistrationTest --- .../SolrCoreLoadRegistrationTest.java | 90 ------------------- 1 file changed, 90 deletions(-) delete mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistrationTest.java diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistrationTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistrationTest.java deleted file mode 100644 index 79d7dd16f..000000000 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistrationTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (C) 2005-2014 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.lifecycle; - -import org.apache.solr.core.SolrConfig; -import org.apache.solr.core.SolrCore; -import org.junit.Test; -import org.xml.sax.InputSource; - -import static org.alfresco.solr.lifecycle.SolrCoreLoadRegistration.isContentStoreInReadOnlyModeFor; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * {@link SolrCoreLoadRegistration} test case. - * - * @author Andrea Gazzarini - * @since 1.5 - */ -public class SolrCoreLoadRegistrationTest -{ - private SolrCore core; - - @Test - public void noReplicationHandlerDefined_thenContentStoreIsInReadWriteMode() throws Exception - { - prepare("solrconfig_no_replication_handler_defined.xml"); - assertFalse("If no replication handler is defined, then we expect to run a RW content store.", isContentStoreInReadOnlyModeFor(core)); - } - - @Test - public void emptyReplicationHandlerDefined_thenContentStoreIsInReadWriteMode() throws Exception - { - prepare("solrconfig_empty_replication_handler.xml"); - assertFalse("If an empty replication handler is defined, then we expect to run a RW content store.", isContentStoreInReadOnlyModeFor(core)); - } - - @Test - public void slaveReplicationHandlerDefinedButDisabled_thenContentStoreIsInReadWriteMode() throws Exception - { - prepare("solrconfig_slave_disabled_replication_handler.xml"); - assertFalse("If a slave replication handler is defined but disabled, then we expect to run a RW content store.", isContentStoreInReadOnlyModeFor(core)); - } - - @Test - public void masterReplicationHandlerDefined_thenContentStoreIsInReadWriteMode() throws Exception - { - prepare("solrconfig_master_replication_handler.xml"); - assertFalse("If a master replication handler is defined but disabled, then we expect to run a RW content store.", isContentStoreInReadOnlyModeFor(core)); - } - - @Test - public void masterReplicationHandlerDefinedButDisabled_thenContentStoreIsInReadWriteMode() throws Exception - { - prepare("solrconfig_master_disabled_replication_handler.xml"); - assertFalse("If a master replication handler is defined but disabled, then we expect to run a RW content store.", isContentStoreInReadOnlyModeFor(core)); - } - - @Test - public void slaveReplicationHandlerDefined_thenContentStoreIsInReadOnlyMode() throws Exception - { - prepare("solrconfig_slave_replication_handler.xml"); - assertTrue("If a slave replication handler is defined, then we expect to run a RO content store.", isContentStoreInReadOnlyModeFor(core)); - } - - private void prepare(String configName) throws Exception - { - core = mock(SolrCore.class); - SolrConfig solrConfig = new SolrConfig(configName, new InputSource(getClass().getResourceAsStream("/test-files/" + configName))); - when(core.getSolrConfig()).thenReturn(solrConfig); - } -} From a9269453ba29eca0f70edbcd24f7e2bc66225273 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Mon, 28 Oct 2019 10:20:36 +0100 Subject: [PATCH 38/76] [contentStoreReplication] toggle contentStore fix after merging from master --- .../java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java | 1 + 1 file changed, 1 insertion(+) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java index c1ec3719d..6f92724ab 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java @@ -119,6 +119,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener boolean trackersHaveBeenEnabled = Boolean.parseBoolean(coreProperties.getProperty("enable.alfresco.tracking", "true")); boolean owningCoreIsSlave = isSlaveModeEnabledFor(core); + contentStore.toggleReadOnlyMode(owningCoreIsSlave); // Guard conditions: if trackers must be disabled then immediately return, we've done here. // Case #1: trackers have been explicitly disabled. From 044fc1462acc18caad4ba0cdf3c9adb7fa88c8af Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Mon, 28 Oct 2019 14:20:55 +0100 Subject: [PATCH 39/76] [contentStoreReplication] get contentStore from AlfrescoCoreAdminHandler --- .../java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java index 6f92724ab..8510859a9 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java @@ -97,7 +97,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener AlfrescoSolrDataModel.getInstance().getDictionaryService(CMISStrictDictionaryService.DEFAULT), AlfrescoSolrDataModel.getInstance().getNamespaceDAO()); - SolrContentStore contentStore = new SolrContentStore(coreContainer.getSolrHome()); + SolrContentStore contentStore = admin.getSolrContentStore(); SolrInformationServer informationServer = new SolrInformationServer(admin, core, repositoryClient, contentStore); coreProperties.putAll(informationServer.getProps()); admin.getInformationServers().put(core.getName(), informationServer); From 3c9aaab2246e078b99cb4faa9cbb5d99fb99fc30 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 31 Oct 2019 09:35:24 +0100 Subject: [PATCH 40/76] [contentStoreReplication] set system property solrhome in SolrDataModel test in order to make it work correctly. --- .../org/alfresco/solr/SolrDataModelTest.java | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrDataModelTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrDataModelTest.java index 17bc9ee73..20575a769 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrDataModelTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrDataModelTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2014 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * @@ -18,17 +18,20 @@ */ package org.alfresco.solr; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -import java.io.InputStream; - import org.alfresco.repo.dictionary.M2Model; import org.alfresco.service.namespace.QName; import org.alfresco.solr.AlfrescoSolrDataModel.FieldUse; import org.alfresco.solr.AlfrescoSolrDataModel.TenantAclIdDbId; +import org.junit.BeforeClass; import org.junit.Test; +import java.io.InputStream; +import java.net.URISyntaxException; + +import static org.alfresco.solr.SolrTestFiles.TEST_FILES_LOCATION; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + /** * @author Andy * @@ -42,6 +45,11 @@ public class SolrDataModelTest private static final QName NAME = QName.createQName("{http://www.alfresco.org/model/cmis/1.0/cs01}name"); private static QName OBJECT_ID = QName.createQName("{http://www.alfresco.org/model/cmis/1.0/cs01}objectId"); + @BeforeClass + public static void initEnvironment(){ + System.setProperty("solr.solr.home", TEST_FILES_LOCATION); + } + @Test public void testDecodeSolr4id() { @@ -90,7 +98,14 @@ public class SolrDataModelTest assertNotNull(modelStream); model = M2Model.createModel(modelStream); dataModel.putModel(model); - + + + try { + System.err.println(cl.getResource("alfresco/model/cmisModel.xml").toURI()); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + assertEquals(2, dataModel.getAlfrescoModels().size()); assertEquals(1, dataModel.getIndexedFieldNamesForProperty(OBJECT_ID).getFields().size()); @@ -144,7 +159,5 @@ public class SolrDataModelTest assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.SORT).getFields().size()); assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.STATS).getFields().size()); assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.SUGGESTION).getFields().size()); - - } } From 6a651d68f396fbac0e4d3f4e2c4db13b201171d7 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 31 Oct 2019 16:22:02 +0100 Subject: [PATCH 41/76] [contentStoreReplication] removed unused content store replication handler with multi stream --- .../alfresco/solr/handler/ReplicationHandler.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java index 85abfb12b..612749613 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java @@ -620,16 +620,11 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw String cfileName = solrParams.get(CONF_FILE_SHORT); String tlogFileName = solrParams.get(TLOG_FILE); - String contentStoreFileName = solrParams.get(CONTENT_STORE_FILE); if (cfileName != null) { rsp.add(FILE_STREAM, new LocalFsConfFileStream(solrParams)); } else if (tlogFileName != null ) { rsp.add(FILE_STREAM, new LocalFsTlogFileStream(solrParams)); - } - else if (contentStoreFileName != null) { - rsp.add(FILE_STREAM, new LocalContentStoreFileStream(solrParams)); - } - else { + } else { rsp.add(FILE_STREAM, new DirectoryFileStream(solrParams)); } } @@ -1502,8 +1497,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw fileName = validateFilenameOrError(params.get(FILE)); cfileName = validateFilenameOrError(params.get(CONF_FILE_SHORT)); tlogFileName = validateFilenameOrError(params.get(TLOG_FILE)); - contentStoreFilename = validateFilenameOrError(params.get(CONTENT_STORE_FILE)); - + sOffset = params.get(OFFSET); sLen = params.get(LEN); compress = params.get(COMPRESSION); @@ -1914,8 +1908,6 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw static final String CONTENT_STORE_FILES = "contentStoreFiles"; - private static final String CONTENT_STORE_FILE = "csf"; - private static final String REPLICATE_AFTER = "replicateAfter"; static final String FILE_STREAM = "filestream"; From 3b27d37b8e25fe1c8079ff8a6423a0f97c7a6f03 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 1 Nov 2019 09:54:04 +0000 Subject: [PATCH 42/76] SEARCH-1763 Rename long running tests to end in IT. --- ...ava => AbstractAlfrescoDistributedIT.java} | 2 +- ...Tests.java => AbstractAlfrescoSolrIT.java} | 7 +++--- ...st.java => AdminHandlerDistributedIT.java} | 2 +- ...inHandlerTest.java => AdminHandlerIT.java} | 3 ++- ...t.java => AlfrescoCoreAdminHandlerIT.java} | 2 +- ...oadTest.java => AlfrescoSolrReloadIT.java} | 5 ++-- .../org/alfresco/solr/AlfrescoSolrUtils.java | 2 +- ...ava => AlfrescoTrackerRegistrationIT.java} | 2 +- ...va => CoresCreateUpdateDistributedIT.java} | 2 +- ...est.java => CoresCreateViaPropertyIT.java} | 2 +- ...nitializer.java => SolrITInitializer.java} | 2 +- ...dTest.java => TemplatesDistributedIT.java} | 2 +- ...a => AlfrescoHighligherDistributedIT.java} | 6 ++--- ...erTest.java => AlfrescoHighlighterIT.java} | 24 ++++--------------- ...er.java => PostingsSolrHighlighterIT.java} | 6 ++--- ...ava => AlfrescoReRankQParserPluginIT.java} | 4 ++-- ...st.java => AlfrescoSolrFingerprintIT.java} | 4 ++-- ...t.java => AlfrescoSolrSpellcheckerIT.java} | 18 ++------------ .../org/alfresco/solr/query/AuthDataLoad.java | 4 ++-- .../{AuthQueryTest.java => AuthQueryIT.java} | 2 +- ...=> DistributedAlfrescoSolrFacetingIT.java} | 6 ++--- ...DistributedAlfrescoSolrFingerPrintIT.java} | 4 ++-- ...istributedAlfrescoSolrSpellcheckerIT.java} | 4 ++-- .../{SolrAuthTest.java => SolrAuthIT.java} | 5 ++-- ...FieldTest.java => UntokenisedFieldIT.java} | 6 ++--- ...Test.java => AbstractQParserPluginIT.java} | 13 ++-------- ...pingTest.java => FieldNameEscapingIT.java} | 2 +- ...erPluginTest.java => QParserPluginIT.java} | 2 +- ...yTest.java => AFTSDefaultTextQueryIT.java} | 2 +- ...nctionTest.java => AFTSDisjunctionIT.java} | 2 +- ...sTest.java => AFTSIdentifierFieldsIT.java} | 2 +- ...geQueryTest.java => AFTSRangeQueryIT.java} | 2 +- ...lerTest.java => AFTSRequestHandlerIT.java} | 2 +- ...est.java => AbstractRequestHandlerIT.java} | 4 ++-- .../{MNTTest.java => MNTIT.java} | 2 +- .../query/cmis/{CmisTest.java => CmisIT.java} | 2 +- .../solr/query/cmis/LoadCMISData.java | 4 ++-- ...uterTest.java => AclModCountRouterIT.java} | 2 +- ...va => AlfrescoSolrTrackerExceptionIT.java} | 6 ++--- ...erTest.java => AlfrescoSolrTrackerIT.java} | 6 ++--- ...ava => AlfrescoSolrTrackerRollbackIT.java} | 6 ++--- ...t.java => AlfrescoSolrTrackerStateIT.java} | 4 ++-- ...TrackerTest.java => CascadeTrackerIT.java} | 4 ++-- ...TrackerTest.java => ContentTrackerIT.java} | 2 +- ...RouterTest.java => DateMonthRouterIT.java} | 2 +- ...istributedAclIdAlfrescoSolrTrackerIT.java} | 4 ++-- ...ava => DistributedAlfrescoSolrJsonIT.java} | 4 ++-- ... => DistributedAlfrescoSolrTrackerIT.java} | 4 ++-- ...DistributedAlfrescoSolrTrackerRaceIT.java} | 4 ++-- ...istributedAlfrescoSolrTrackerStateIT.java} | 4 ++-- ....java => DistributedCascadeTrackerIT.java} | 4 ++-- ...DistributedDateAbstractSolrTrackerIT.java} | 4 ++-- ...ibutedDateMonthAlfrescoSolrTrackerIT.java} | 6 ++--- ...utedDateQuarterAlfrescoSolrTrackerIT.java} | 2 +- ...edDateSplitYearAlfrescoSolrTrackerIT.java} | 4 ++-- ...ibutedDbidRangeAlfrescoSolrTrackerIT.java} | 6 ++--- ...ExpandDbidRangeAlfrescoSolrTrackerIT.java} | 6 ++--- ...citShardIdWithStaticPropertyRouterIT.java} | 4 ++-- ...ributedExplicitShardRoutingTrackerIT.java} | 6 ++--- ...edPropertyBasedAlfrescoSolrTrackerIT.java} | 6 ++--- ...outerTest.java => ExplicitIDRouterIT.java} | 2 +- ...t.java => ExplicitIDWithLRISRouterIT.java} | 2 +- ...elTrackerTest.java => ModelTrackerIT.java} | 2 +- ...yRouterTest.java => PropertyRouterIT.java} | 2 +- ...rTest.java => CachedDocTransformerIT.java} | 6 ++--- ...shFilterTest.java => MinHashFilterIT.java} | 2 +- ...java => AlfrescoLukeRequestHandlerIT.java} | 2 +- 67 files changed, 120 insertions(+), 163 deletions(-) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{AbstractAlfrescoDistributedTest.java => AbstractAlfrescoDistributedIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{AbstractAlfrescoSolrTests.java => AbstractAlfrescoSolrIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{AdminHandlerDistributedTest.java => AdminHandlerDistributedIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{AdminHandlerTest.java => AdminHandlerIT.java} (96%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{AlfrescoCoreAdminHandlerTest.java => AlfrescoCoreAdminHandlerIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{AlfrescoSolrReloadTest.java => AlfrescoSolrReloadIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{AlfrescoTrackerRegistrationTest.java => AlfrescoTrackerRegistrationIT.java} (96%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{CoresCreateUpdateDistributedTest.java => CoresCreateUpdateDistributedIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{CoresCreateViaPropertyTest.java => CoresCreateViaPropertyIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{SolrTestInitializer.java => SolrITInitializer.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/{TemplatesDistributedTest.java => TemplatesDistributedIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/{AlfrescoHighligherDistributedTest.java => AlfrescoHighligherDistributedIT.java} (96%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/{AlfrescoHighlighterTest.java => AlfrescoHighlighterIT.java} (95%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/{TestPostingsSolrHighlighter.java => PostingsSolrHighlighterIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/{AlfrescoReRankQParserPluginTest.java => AlfrescoReRankQParserPluginIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/{AlfrescoSolrFingerprintTest.java => AlfrescoSolrFingerprintIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/{AlfrescoSolrSpellcheckerTest.java => AlfrescoSolrSpellcheckerIT.java} (88%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/{AuthQueryTest.java => AuthQueryIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/{DistributedAlfrescoSolrFacetingTest.java => DistributedAlfrescoSolrFacetingIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/{DistributedAlfrescoSolrFingerPrintTest.java => DistributedAlfrescoSolrFingerPrintIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/{DistributedAlfrescoSolrSpellcheckerTest.java => DistributedAlfrescoSolrSpellcheckerIT.java} (96%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/{SolrAuthTest.java => SolrAuthIT.java} (95%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/{UntokenisedFieldTest.java => UntokenisedFieldIT.java} (94%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/{AbstractQParserPluginTest.java => AbstractQParserPluginIT.java} (74%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/{FieldNameEscapingTest.java => FieldNameEscapingIT.java} (94%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/{QParserPluginTest.java => QParserPluginIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/{AFTSDefaultTextQueryTest.java => AFTSDefaultTextQueryIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/{AFTSDisjunctionTest.java => AFTSDisjunctionIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/{AFTSIdentifierFieldsTest.java => AFTSIdentifierFieldsIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/{AFTSRangeQueryTest.java => AFTSRangeQueryIT.java} (98%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/{AFTSRequestHandlerTest.java => AFTSRequestHandlerIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/{AbstractRequestHandlerTest.java => AbstractRequestHandlerIT.java} (95%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/{MNTTest.java => MNTIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/{CmisTest.java => CmisIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{AclModCountRouterTest.java => AclModCountRouterIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{AlfrescoSolrTrackerExceptionTest.java => AlfrescoSolrTrackerExceptionIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{AlfrescoSolrTrackerTest.java => AlfrescoSolrTrackerIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{AlfrescoSolrTrackerRollbackTest.java => AlfrescoSolrTrackerRollbackIT.java} (98%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{AlfrescoSolrTrackerStateTest.java => AlfrescoSolrTrackerStateIT.java} (98%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{CascadeTrackerTest.java => CascadeTrackerIT.java} (98%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{ContentTrackerTest.java => ContentTrackerIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DateMonthRouterTest.java => DateMonthRouterIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedAclIdAlfrescoSolrTrackerTest.java => DistributedAclIdAlfrescoSolrTrackerIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedAlfrescoSolrJsonTest.java => DistributedAlfrescoSolrJsonIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedAlfrescoSolrTrackerTest.java => DistributedAlfrescoSolrTrackerIT.java} (98%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedAlfrescoSolrTrackerRaceTest.java => DistributedAlfrescoSolrTrackerRaceIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedAlfrescoSolrTrackerStateTest.java => DistributedAlfrescoSolrTrackerStateIT.java} (98%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedCascadeTrackerTest.java => DistributedCascadeTrackerIT.java} (98%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedDateAbstractSolrTrackerTest.java => DistributedDateAbstractSolrTrackerIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedDateMonthAlfrescoSolrTrackerTest.java => DistributedDateMonthAlfrescoSolrTrackerIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedDateQuarterAlfrescoSolrTrackerTest.java => DistributedDateQuarterAlfrescoSolrTrackerIT.java} (95%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedDateSplitYearAlfrescoSolrTrackerTest.java => DistributedDateSplitYearAlfrescoSolrTrackerIT.java} (93%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedDbidRangeAlfrescoSolrTrackerTest.java => DistributedDbidRangeAlfrescoSolrTrackerIT.java} (96%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedExpandDbidRangeAlfrescoSolrTrackerTest.java => DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java} (98%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedExplicitShardIdWithStaticPropertyRouterTest.java => DistributedExplicitShardIdWithStaticPropertyRouterIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedExplicitShardRoutingTrackerTest.java => DistributedExplicitShardRoutingTrackerIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedPropertyBasedAlfrescoSolrTrackerTest.java => DistributedPropertyBasedAlfrescoSolrTrackerIT.java} (97%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{ExplicitIDRouterTest.java => ExplicitIDRouterIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{ExplicitIDWithLRISRouterTest.java => ExplicitIDWithLRISRouterIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{ModelTrackerTest.java => ModelTrackerIT.java} (96%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{PropertyRouterTest.java => PropertyRouterIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/{CachedDocTransformerTest.java => CachedDocTransformerIT.java} (98%) rename search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/{MinHashFilterTest.java => MinHashFilterIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/apache/solr/handler/component/{AlfrescoLukeRequestHandlerTest.java => AlfrescoLukeRequestHandlerIT.java} (99%) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java index 7fcef7193..14a9a805d 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java @@ -87,7 +87,7 @@ import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_VERSI * @author Michael Suzuki */ @ThreadLeakLingering(linger = 5000) -public abstract class AbstractAlfrescoDistributedTest extends SolrTestInitializer +public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer { protected static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java index a7647a399..d9ee37eaa 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrTests.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java @@ -27,7 +27,6 @@ import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.client.SOLRAPIQueueClient; import org.alfresco.solr.client.Transaction; import org.alfresco.solr.tracker.Tracker; -import org.alfresco.util.SearchLanguageConversion; import org.apache.chemistry.opencmis.commons.impl.json.JSONArray; import org.apache.chemistry.opencmis.commons.impl.json.JSONObject; import org.apache.chemistry.opencmis.commons.impl.json.JSONValue; @@ -84,17 +83,17 @@ import static org.junit.Assert.assertEquals; /** * Base class that provides the solr test harness. - * This is used to manage the embedded solr used for unit and integration testing. + * This is used to manage the embedded solr used for integration testing. * The abstract also provides helper method that interacts with the * embedded solr. * * @author Michael Suzuki * */ -public abstract class AbstractAlfrescoSolrTests implements SolrTestFiles, AlfrescoSolrConstants +public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoSolrConstants { static AlfrescoCoreAdminHandler admin; - private static Log LOG = LogFactory.getLog(AbstractAlfrescoSolrTests.class); + private static Log LOG = LogFactory.getLog(AbstractAlfrescoSolrIT.class); private static boolean CORE_NOT_YET_CREATED = true; /** diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java index 95b98db09..1f6255b13 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java @@ -43,7 +43,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.*; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class AdminHandlerDistributedTest extends AbstractAlfrescoDistributedTest +public class AdminHandlerDistributedIT extends AbstractAlfrescoDistributedIT { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); final String JETTY_SERVER_ID = this.getClass().getSimpleName(); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java similarity index 96% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java index 872fea72c..140b6b1db 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java @@ -11,7 +11,8 @@ import org.junit.Test; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class AdminHandlerTest extends AbstractAlfrescoSolrTests { +public class AdminHandlerIT extends AbstractAlfrescoSolrIT +{ static CoreAdminHandler admin; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java index 955f365b5..5e264adb3 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java @@ -60,7 +60,7 @@ import org.mockito.junit.MockitoJUnitRunner; /** Unit tests for {@link org.alfresco.solr.AlfrescoCoreAdminHandler}. */ @RunWith(MockitoJUnitRunner.class) -public class AlfrescoCoreAdminHandlerTest +public class AlfrescoCoreAdminHandlerIT { /** The string representing a transaction report. */ private static final String TXREPORT = "TXREPORT"; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java index c56138ed0..70500adf6 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java @@ -37,8 +37,9 @@ import org.quartz.SchedulerException; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class AlfrescoSolrReloadTest extends AbstractAlfrescoSolrTests { - private static Log logger = LogFactory.getLog(org.alfresco.solr.tracker.AlfrescoSolrTrackerTest.class); +public class AlfrescoSolrReloadIT extends AbstractAlfrescoSolrIT +{ + private static Log logger = LogFactory.getLog(org.alfresco.solr.tracker.AlfrescoSolrTrackerIT.class); @BeforeClass public static void beforeClass() throws Exception { diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java index 174939124..03f55926f 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java @@ -77,7 +77,7 @@ import org.alfresco.service.cmr.repository.ChildAssociationRef; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.service.namespace.QName; -import org.alfresco.solr.AbstractAlfrescoSolrTests.SolrServletRequest; +import org.alfresco.solr.AbstractAlfrescoSolrIT.SolrServletRequest; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationIT.java similarity index 96% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationIT.java index cbb1c1e36..b679ec5fe 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoTrackerRegistrationIT.java @@ -30,7 +30,7 @@ import org.junit.BeforeClass; import org.junit.Test; @SolrTestCaseJ4.SuppressSSL -public class AlfrescoTrackerRegistrationTest extends AbstractAlfrescoSolrTests +public class AlfrescoTrackerRegistrationIT extends AbstractAlfrescoSolrIT { @BeforeClass public static void beforeClass() throws Exception { diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateUpdateDistributedTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateUpdateDistributedIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateUpdateDistributedTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateUpdateDistributedIT.java index 2081c770f..feb54a648 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateUpdateDistributedTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateUpdateDistributedIT.java @@ -51,7 +51,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.getCore; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class CoresCreateUpdateDistributedTest extends AbstractAlfrescoDistributedTest +public class CoresCreateUpdateDistributedIT extends AbstractAlfrescoDistributedIT { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); final static String JETTY_SERVER_ID = "CoresCreateUpdateDistributedTest"; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateViaPropertyTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateViaPropertyIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateViaPropertyTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateViaPropertyIT.java index 2bc699dd5..82d128394 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateViaPropertyTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateViaPropertyIT.java @@ -40,7 +40,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.getCore; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class CoresCreateViaPropertyTest extends AbstractAlfrescoDistributedTest +public class CoresCreateViaPropertyIT extends AbstractAlfrescoDistributedIT { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); final static String JETTY_SERVER_ID = "CoresCreateViaPropertyTest"; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestInitializer.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index e9019df77..08be29603 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrTestInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -69,7 +69,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.createCoreUsingTemplate; * @author Michael Suzuki */ @ThreadLeakLingering(linger = 5000) -public abstract class SolrTestInitializer extends SolrTestCaseJ4 +public abstract class SolrITInitializer extends SolrTestCaseJ4 { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/TemplatesDistributedTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/TemplatesDistributedIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/TemplatesDistributedTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/TemplatesDistributedIT.java index 6ed551911..b7db5c77a 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/TemplatesDistributedTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/TemplatesDistributedIT.java @@ -40,7 +40,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.*; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class TemplatesDistributedTest extends AbstractAlfrescoDistributedTest +public class TemplatesDistributedIT extends AbstractAlfrescoDistributedIT { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); final static String JETTY_SERVER_ID = "TemplatesDistributedTest"; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighligherDistributedTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighligherDistributedIT.java similarity index 96% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighligherDistributedTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighligherDistributedIT.java index 62c0397d8..4eb9cae4c 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighligherDistributedTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighligherDistributedIT.java @@ -18,7 +18,7 @@ */ package org.alfresco.solr.highlight; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.util.LuceneTestCase; @@ -35,9 +35,9 @@ import org.junit.Test; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class AlfrescoHighligherDistributedTest extends AbstractAlfrescoDistributedTest +public class AlfrescoHighligherDistributedIT extends AbstractAlfrescoDistributedIT { - private static Log logger = LogFactory.getLog(AlfrescoHighligherDistributedTest.class); + private static Log logger = LogFactory.getLog(AlfrescoHighligherDistributedIT.class); @BeforeClass private static void initData() throws Throwable diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterIT.java similarity index 95% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterIT.java index b63746f64..c19a8ee68 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighlighterIT.java @@ -22,13 +22,9 @@ package org.alfresco.solr.highlight; import java.util.Map; import org.alfresco.model.ContentModel; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; -import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.namespace.QName; -import org.alfresco.solr.AbstractAlfrescoSolrTests; -import org.alfresco.solr.AlfrescoCoreAdminHandler; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.alfresco.solr.client.*; -import org.alfresco.solr.dataload.TestDataProvider; -import org.alfresco.util.Pair; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.index.Term; @@ -37,39 +33,27 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.params.HighlightParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.response.BasicResultContext; -import org.apache.solr.response.SolrQueryResponse; -import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.util.Random; import static com.google.common.collect.ImmutableMap.of; import static java.util.Arrays.asList; -import static java.util.stream.IntStream.range; + import static junit.framework.TestCase.assertTrue; -import static org.alfresco.model.ContentModel.PROP_CREATOR; -import static org.alfresco.model.ContentModel.PROP_RATING_SCHEME; -import static org.alfresco.model.ContentModel.TYPE_CONTENT; import static org.alfresco.solr.AlfrescoSolrUtils.*; -import static org.apache.solr.SolrJettyTestBase.jetty; import static org.junit.Assert.assertNotNull; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class AlfrescoHighlighterTest extends AbstractAlfrescoSolrTests +public class AlfrescoHighlighterIT extends AbstractAlfrescoSolrIT { - private static Log logger = LogFactory.getLog(AlfrescoHighlighterTest.class); + private static Log logger = LogFactory.getLog(AlfrescoHighlighterIT.class); private static long MAX_WAIT_TIME = 80000; @BeforeClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/TestPostingsSolrHighlighter.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/PostingsSolrHighlighterIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/TestPostingsSolrHighlighter.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/PostingsSolrHighlighterIT.java index bc50a73c1..2b54f3af2 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/TestPostingsSolrHighlighter.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/PostingsSolrHighlighterIT.java @@ -16,12 +16,10 @@ */ package org.alfresco.solr.highlight; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.handler.component.AlfrescoSolrHighlighter; import org.apache.solr.handler.component.HighlightComponent; -import org.apache.solr.highlight.PostingsSolrHighlighter; import org.apache.solr.highlight.SolrHighlighter; import org.apache.solr.schema.IndexSchema; import org.junit.Before; @@ -31,7 +29,7 @@ import org.junit.Test; import static junit.framework.TestCase.*; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class TestPostingsSolrHighlighter extends AbstractAlfrescoSolrTests +public class PostingsSolrHighlighterIT extends AbstractAlfrescoSolrIT { @BeforeClass public static void beforeClass() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoReRankQParserPluginTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoReRankQParserPluginIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoReRankQParserPluginTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoReRankQParserPluginIT.java index 34d1a2d6b..1797a5c17 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoReRankQParserPluginTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoReRankQParserPluginIT.java @@ -22,7 +22,7 @@ package org.alfresco.solr.query; import java.util.ArrayList; import java.util.List; import java.util.Random; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; @@ -38,7 +38,7 @@ import org.junit.Test; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class AlfrescoReRankQParserPluginTest extends AbstractAlfrescoSolrTests +public class AlfrescoReRankQParserPluginIT extends AbstractAlfrescoSolrIT { @BeforeClass public static void beforeClass() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrFingerprintTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrFingerprintIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrFingerprintTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrFingerprintIT.java index 96e1285c0..d9f0694e2 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrFingerprintTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrFingerprintIT.java @@ -21,7 +21,7 @@ package org.alfresco.solr.query; import org.alfresco.model.ContentModel; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; @@ -51,7 +51,7 @@ import static java.util.Collections.singletonList; import static java.util.stream.IntStream.range; import static org.alfresco.solr.AlfrescoSolrUtils.*; -public class AlfrescoSolrFingerprintTest extends AbstractAlfrescoSolrTests +public class AlfrescoSolrFingerprintIT extends AbstractAlfrescoSolrIT { private static long MAX_WAIT_TIME = 80000; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSpellcheckerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSpellcheckerIT.java similarity index 88% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSpellcheckerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSpellcheckerIT.java index 98a723d2a..4d6ad4257 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSpellcheckerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSpellcheckerIT.java @@ -19,31 +19,17 @@ package org.alfresco.solr.query; -import static org.junit.Assert.*; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.RefCount; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.ResultContext; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.search.DocIterator; -import org.apache.solr.search.DocList; -import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.util.RefCounted; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class AlfrescoSolrSpellcheckerTest extends AbstractAlfrescoSolrTests +public class AlfrescoSolrSpellcheckerIT extends AbstractAlfrescoSolrIT { @BeforeClass public static void beforeClass() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthDataLoad.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthDataLoad.java index 3a201b545..0b86af558 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthDataLoad.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthDataLoad.java @@ -34,7 +34,7 @@ import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.alfresco.solr.client.ContentPropertyValue; import org.alfresco.solr.client.MLTextPropertyValue; import org.alfresco.solr.client.PropertyValue; @@ -46,7 +46,7 @@ import org.junit.BeforeClass; * @author Michael Suzuki * */ -public class AuthDataLoad extends AbstractAlfrescoSolrTests +public class AuthDataLoad extends AbstractAlfrescoSolrIT { static int count = 100; static long maxReader = 1000; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthQueryTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthQueryIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthQueryTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthQueryIT.java index 00cc9c06a..55a93e309 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthQueryTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthQueryIT.java @@ -39,7 +39,7 @@ import org.junit.Test; * @author Michael Suzuki * */ -public class AuthQueryTest extends AuthDataLoad +public class AuthQueryIT extends AuthDataLoad { @Test public void checkAuth() diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFacetingTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFacetingIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFacetingTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFacetingIT.java index 457497cd1..13f28d874 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFacetingTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFacetingIT.java @@ -18,7 +18,7 @@ */ package org.alfresco.solr.query; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.response.FacetField; @@ -37,12 +37,12 @@ import static org.hamcrest.core.Is.is; @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({ "Appending", "Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Lucene43", "Lucene44", "Lucene45", "Lucene46", "Lucene47", "Lucene48", - "Lucene49" }) public class DistributedAlfrescoSolrFacetingTest extends AbstractAlfrescoDistributedTest + "Lucene49" }) public class DistributedAlfrescoSolrFacetingIT extends AbstractAlfrescoDistributedIT { @BeforeClass private static void initData() throws Throwable { - initSolrServers(2, "DistributedAlfrescoSolrFacetingTest", null); + initSolrServers(2, "DistributedAlfrescoSolrFacetingIT", null); indexSampleDocumentsForFacetingMincount(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFingerPrintTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFingerPrintIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFingerPrintTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFingerPrintIT.java index 980ad45db..dfd4326c8 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFingerPrintTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFingerPrintIT.java @@ -34,7 +34,7 @@ import java.util.Random; import org.alfresco.model.ContentModel; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; @@ -61,7 +61,7 @@ import org.junit.Test; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedAlfrescoSolrFingerPrintTest extends AbstractAlfrescoDistributedTest +public class DistributedAlfrescoSolrFingerPrintIT extends AbstractAlfrescoDistributedIT { private static long MAX_WAIT_TIME = 80000; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrSpellcheckerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrSpellcheckerIT.java similarity index 96% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrSpellcheckerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrSpellcheckerIT.java index ea05e052a..1561564ed 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrSpellcheckerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrSpellcheckerIT.java @@ -18,7 +18,7 @@ */ package org.alfresco.solr.query; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.response.QueryResponse; @@ -32,7 +32,7 @@ import org.junit.Test; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedAlfrescoSolrSpellcheckerTest extends AbstractAlfrescoDistributedTest +public class DistributedAlfrescoSolrSpellcheckerIT extends AbstractAlfrescoDistributedIT { @BeforeClass private static void initData() throws Throwable diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/SolrAuthTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/SolrAuthIT.java similarity index 95% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/SolrAuthTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/SolrAuthIT.java index 2b4b28fe1..0491c2f4a 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/SolrAuthTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/SolrAuthIT.java @@ -20,19 +20,18 @@ package org.alfresco.solr.query; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.params.ModifiableSolrParams; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.runners.Parameterized.Parameters; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class SolrAuthTest extends AbstractAlfrescoSolrTests +public class SolrAuthIT extends AbstractAlfrescoSolrIT { @BeforeClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/UntokenisedFieldTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/UntokenisedFieldIT.java similarity index 94% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/UntokenisedFieldTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/UntokenisedFieldIT.java index 0bfbf3614..694364efd 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/UntokenisedFieldTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/UntokenisedFieldIT.java @@ -18,9 +18,7 @@ */ package org.alfresco.solr.query; -import org.alfresco.model.ContentModel; -import org.alfresco.service.cmr.search.GeneralHighlightParameters; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.junit.Before; import org.junit.Test; @@ -32,7 +30,7 @@ import org.junit.Test; * @author Michael Suzuki * */ -public class UntokenisedFieldTest extends AbstractAlfrescoSolrTests +public class UntokenisedFieldIT extends AbstractAlfrescoSolrIT { String nodeRef = "workspace://SpacesStore/00000000-0000-1-4731-76966678"; String nodeRefS = "noderef@s_@mytest" ; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/AbstractQParserPluginTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/AbstractQParserPluginIT.java similarity index 74% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/AbstractQParserPluginTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/AbstractQParserPluginIT.java index 177760071..35520530a 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/AbstractQParserPluginTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/AbstractQParserPluginIT.java @@ -18,25 +18,16 @@ */ package org.alfresco.solr.query.afts.qparser; -import static java.util.Arrays.asList; -import static java.util.stream.IntStream.range; - -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.junit.BeforeClass; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * Supertype layer for all AFTS QParser tests. * * @author Andrea Gazzarini */ -public abstract class AbstractQParserPluginTest extends AbstractAlfrescoSolrTests +public abstract class AbstractQParserPluginIT extends AbstractAlfrescoSolrIT { @BeforeClass public static void spinUpSolr() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/FieldNameEscapingTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/FieldNameEscapingIT.java similarity index 94% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/FieldNameEscapingTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/FieldNameEscapingIT.java index 6485fd022..f2d103787 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/FieldNameEscapingTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/FieldNameEscapingIT.java @@ -24,7 +24,7 @@ import org.alfresco.util.ISO9075; import org.junit.BeforeClass; import org.junit.Test; -public class FieldNameEscapingTest extends AbstractQParserPluginTest implements QueryConstants +public class FieldNameEscapingIT extends AbstractQParserPluginIT implements QueryConstants { private static TestDataProvider DATASETS_PROVIDER; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/QParserPluginTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/QParserPluginIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/QParserPluginTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/QParserPluginIT.java index 32c889d30..9a15161c8 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/QParserPluginTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/qparser/QParserPluginIT.java @@ -43,7 +43,7 @@ import org.junit.BeforeClass; import org.junit.Test; @SolrTestCaseJ4.SuppressSSL -public class QParserPluginTest extends AbstractQParserPluginTest implements QueryConstants +public class QParserPluginIT extends AbstractQParserPluginIT implements QueryConstants { /** The UTC time zone. */ private static final TimeZone UTC = TimeZone.getTimeZone("UTC"); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryIT.java index 6791bbf32..e4d3ad755 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryIT.java @@ -49,7 +49,7 @@ import static com.google.common.collect.ImmutableMap.of; * This test set checks that in all these query types the default fields are involved in the search. * */ -public class AFTSDefaultTextQueryTest extends AbstractRequestHandlerTest +public class AFTSDefaultTextQueryIT extends AbstractRequestHandlerIT { @BeforeClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDisjunctionTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDisjunctionIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDisjunctionTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDisjunctionIT.java index 8462d00e3..3037a3e2d 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDisjunctionTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDisjunctionIT.java @@ -39,7 +39,7 @@ import org.junit.Test; * * @author msuzuki */ -public class AFTSDisjunctionTest extends AbstractRequestHandlerTest +public class AFTSDisjunctionIT extends AbstractRequestHandlerIT { @BeforeClass public static void beforeClass() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSIdentifierFieldsTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSIdentifierFieldsIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSIdentifierFieldsTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSIdentifierFieldsIT.java index 2dc910b8c..e4eac23b2 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSIdentifierFieldsTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSIdentifierFieldsIT.java @@ -43,7 +43,7 @@ import java.util.Map; * @author eporciani * @author agazzarini */ -public class AFTSIdentifierFieldsTest extends AbstractRequestHandlerTest +public class AFTSIdentifierFieldsIT extends AbstractRequestHandlerIT { @BeforeClass public static void beforeClass() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRangeQueryTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRangeQueryIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRangeQueryTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRangeQueryIT.java index ed38a5006..ba7dc7f13 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRangeQueryTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRangeQueryIT.java @@ -41,7 +41,7 @@ import java.util.Map; * * @author elia */ -public class AFTSRangeQueryTest extends AbstractRequestHandlerTest +public class AFTSRangeQueryIT extends AbstractRequestHandlerIT { @BeforeClass public static void beforeClass() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRequestHandlerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRequestHandlerIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRequestHandlerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRequestHandlerIT.java index b6f818791..131bd8dce 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRequestHandlerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSRequestHandlerIT.java @@ -20,7 +20,7 @@ import java.util.Locale; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class AFTSRequestHandlerTest extends AbstractRequestHandlerTest implements QueryConstants +public class AFTSRequestHandlerIT extends AbstractRequestHandlerIT implements QueryConstants { private static TestDataProvider DATASETS_PROVIDER; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerIT.java similarity index 95% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerIT.java index 3a4bf2681..929b0d30e 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerIT.java @@ -1,6 +1,6 @@ package org.alfresco.solr.query.afts.requestHandler; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.junit.BeforeClass; import java.util.ArrayList; @@ -12,7 +12,7 @@ import java.util.stream.Stream; import static java.util.Arrays.asList; import static java.util.stream.IntStream.range; -public abstract class AbstractRequestHandlerTest extends AbstractAlfrescoSolrTests +public abstract class AbstractRequestHandlerIT extends AbstractAlfrescoSolrIT { @BeforeClass public static void spinUpSolr() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/MNTTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/MNTIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/MNTTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/MNTIT.java index a7b0dd5df..613e2da1e 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/MNTTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/MNTIT.java @@ -9,7 +9,7 @@ import org.junit.Test; * * @author Andrea Gazzarini */ -public class MNTTest extends AbstractRequestHandlerTest +public class MNTIT extends AbstractRequestHandlerIT { @BeforeClass public static void loadData() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/CmisTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/CmisIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/CmisTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/CmisIT.java index 570a88cc9..6b05203f2 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/CmisTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/CmisIT.java @@ -31,7 +31,7 @@ import org.junit.Test; * @author Michael Suzuki * */ -public class CmisTest extends LoadCMISData +public class CmisIT extends LoadCMISData { @Before public void setup() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/LoadCMISData.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/LoadCMISData.java index 46082dac9..f64efe0a5 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/LoadCMISData.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/LoadCMISData.java @@ -37,7 +37,7 @@ import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.client.ContentPropertyValue; import org.alfresco.solr.client.MLTextPropertyValue; @@ -51,7 +51,7 @@ import org.junit.BeforeClass; * @author Michael Suzuki * */ -public class LoadCMISData extends AbstractAlfrescoSolrTests +public class LoadCMISData extends AbstractAlfrescoSolrIT { protected static NodeRef testCMISContent00NodeRef; protected static NodeRef testCMISRootNodeRef; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclModCountRouterTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclModCountRouterIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclModCountRouterTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclModCountRouterIT.java index c903cd8f3..451b29ed6 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclModCountRouterTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclModCountRouterIT.java @@ -39,7 +39,7 @@ import java.util.Map; import java.util.Properties; @RunWith(MockitoJUnitRunner.class) -public class AclModCountRouterTest +public class AclModCountRouterIT { private DocRouter router; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java index 8a728a519..43d779f8f 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java @@ -36,7 +36,7 @@ import org.alfresco.model.ContentModel; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.repository.StoreRef; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; @@ -62,9 +62,9 @@ import org.junit.Test; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class AlfrescoSolrTrackerExceptionTest extends AbstractAlfrescoSolrTests +public class AlfrescoSolrTrackerExceptionIT extends AbstractAlfrescoSolrIT { - private static Log logger = LogFactory.getLog(AlfrescoSolrTrackerTest.class); + private static Log logger = LogFactory.getLog(AlfrescoSolrTrackerExceptionIT.class); private static long MAX_WAIT_TIME = 80000; @BeforeClass public static void beforeClass() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java index c1e8c70fb..749e82c03 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java @@ -37,7 +37,7 @@ import org.alfresco.model.ContentModel; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.repository.StoreRef; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; @@ -64,9 +64,9 @@ import org.junit.Test; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class AlfrescoSolrTrackerTest extends AbstractAlfrescoSolrTests +public class AlfrescoSolrTrackerIT extends AbstractAlfrescoSolrIT { - private static Log logger = LogFactory.getLog(AlfrescoSolrTrackerTest.class); + private static Log logger = LogFactory.getLog(AlfrescoSolrTrackerIT.class); private static long MAX_WAIT_TIME = 80000; @BeforeClass public static void beforeClass() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerRollbackTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerRollbackIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerRollbackTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerRollbackIT.java index d2ea46e05..2313586ac 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerRollbackTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerRollbackIT.java @@ -32,7 +32,7 @@ import java.util.Collection; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.alfresco.solr.client.Acl; @@ -58,9 +58,9 @@ import org.junit.Test; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class AlfrescoSolrTrackerRollbackTest extends AbstractAlfrescoSolrTests +public class AlfrescoSolrTrackerRollbackIT extends AbstractAlfrescoSolrIT { - private static Log logger = LogFactory.getLog(AlfrescoSolrTrackerTest.class); + private static Log logger = LogFactory.getLog(AlfrescoSolrTrackerIT.class); private static long MAX_WAIT_TIME = 80000; @BeforeClass public static void beforeClass() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java index d37d8ab99..119aab737 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java @@ -36,7 +36,7 @@ import static org.junit.Assert.assertNotEquals; import org.alfresco.repo.index.shard.ShardState; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; @@ -70,7 +70,7 @@ import java.util.stream.Collectors; * * @author agazzarini */ -public class AlfrescoSolrTrackerStateTest extends AbstractAlfrescoSolrTests +public class AlfrescoSolrTrackerStateIT extends AbstractAlfrescoSolrIT { @BeforeClass public static void beforeClass() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerIT.java index 16f1b0c56..00c34cfa4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerIT.java @@ -35,7 +35,7 @@ import org.alfresco.model.ContentModel; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.repository.StoreRef; -import org.alfresco.solr.AbstractAlfrescoSolrTests; +import org.alfresco.solr.AbstractAlfrescoSolrIT; import org.alfresco.solr.client.*; import org.apache.lucene.index.Term; import org.apache.lucene.search.*; @@ -52,7 +52,7 @@ import java.util.List; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class CascadeTrackerTest extends AbstractAlfrescoSolrTests +public class CascadeTrackerIT extends AbstractAlfrescoSolrIT { private static long MAX_WAIT_TIME = 80000; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java index e3d6112da..517a5f241 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java @@ -38,7 +38,7 @@ import org.mockito.Spy; import org.mockito.runners.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ContentTrackerTest +public class ContentTrackerIT { private ContentTracker contentTracker; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DateMonthRouterTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DateMonthRouterIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DateMonthRouterTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DateMonthRouterIT.java index 2432966d5..03cf95823 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DateMonthRouterTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DateMonthRouterIT.java @@ -46,7 +46,7 @@ import java.util.Properties; import java.util.Random; @RunWith(MockitoJUnitRunner.class) -public class DateMonthRouterTest +public class DateMonthRouterIT { private Random randomizer = new Random(); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerIT.java index f177f915d..8c268e01a 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerIT.java @@ -19,7 +19,7 @@ package org.alfresco.solr.tracker; import org.alfresco.repo.index.shard.ShardMethodEnum; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; @@ -57,7 +57,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.list; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedAclIdAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest +public class DistributedAclIdAlfrescoSolrTrackerIT extends AbstractAlfrescoDistributedIT { @BeforeClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrJsonTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrJsonIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrJsonTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrJsonIT.java index 1a86633e2..c0d0c14ec 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrJsonTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrJsonIT.java @@ -19,7 +19,7 @@ package org.alfresco.solr.tracker; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.Node; @@ -50,7 +50,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedAlfrescoSolrJsonTest extends AbstractAlfrescoDistributedTest +public class DistributedAlfrescoSolrJsonIT extends AbstractAlfrescoDistributedIT { @BeforeClass private static void initData() throws Throwable diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerIT.java index b80d8793e..8b0f610f9 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerIT.java @@ -19,7 +19,7 @@ package org.alfresco.solr.tracker; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; @@ -56,7 +56,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.list; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest +public class DistributedAlfrescoSolrTrackerIT extends AbstractAlfrescoDistributedIT { @BeforeClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java index 65bab2295..01290fcc7 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java @@ -19,7 +19,7 @@ package org.alfresco.solr.tracker; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; @@ -54,7 +54,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedAlfrescoSolrTrackerRaceTest extends AbstractAlfrescoDistributedTest +public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistributedIT { @BeforeClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerStateTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerStateIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerStateTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerStateIT.java index 58e697c9d..e33447951 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerStateTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerStateIT.java @@ -33,7 +33,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; import org.alfresco.repo.index.shard.ShardState; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; @@ -68,7 +68,7 @@ import java.util.stream.Collectors; * @author agazzarini */ @SolrTestCaseJ4.SuppressSSL -public class DistributedAlfrescoSolrTrackerStateTest extends AbstractAlfrescoDistributedTest +public class DistributedAlfrescoSolrTrackerStateIT extends AbstractAlfrescoDistributedIT { @BeforeClass private static void initData() throws Throwable diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java index cba592b86..6e5fbe39f 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java @@ -30,7 +30,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; import static org.carrot2.shaded.guava.common.collect.ImmutableList.of; import org.alfresco.model.ContentModel; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; @@ -55,7 +55,7 @@ import java.util.Properties; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedCascadeTrackerTest extends AbstractAlfrescoDistributedTest +public class DistributedCascadeTrackerIT extends AbstractAlfrescoDistributedIT { private Node parentFolder; private NodeMetaData parentFolderMetadata; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerIT.java index 6aacd0282..1cdaa1cc2 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerIT.java @@ -21,7 +21,7 @@ package org.alfresco.solr.tracker; import org.alfresco.model.ContentModel; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; import org.alfresco.service.namespace.QName; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; @@ -63,7 +63,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public abstract class DistributedDateAbstractSolrTrackerTest extends AbstractAlfrescoDistributedTest +public abstract class DistributedDateAbstractSolrTrackerIT extends AbstractAlfrescoDistributedIT { @Test public void testDateMonth() throws Exception diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerIT.java index e0a569b4a..8dcbf6e5e 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerIT.java @@ -33,7 +33,7 @@ import org.alfresco.model.ContentModel; import org.alfresco.repo.index.shard.ShardMethodEnum; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; import org.alfresco.service.namespace.QName; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.Acl; @@ -63,13 +63,13 @@ import java.util.Properties; import java.util.TimeZone; @SolrTestCaseJ4.SuppressSSL -public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest +public class DistributedDateMonthAlfrescoSolrTrackerIT extends AbstractAlfrescoDistributedIT { @BeforeClass @SuppressWarnings("unused") public static void initData() throws Throwable { - initSolrServers(5, DistributedDateMonthAlfrescoSolrTrackerTest.class.getSimpleName(), getShardMethod()); + initSolrServers(5, DistributedDateMonthAlfrescoSolrTrackerIT.class.getSimpleName(), getShardMethod()); } @AfterClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateQuarterAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateQuarterAlfrescoSolrTrackerIT.java similarity index 95% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateQuarterAlfrescoSolrTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateQuarterAlfrescoSolrTrackerIT.java index 0c8c061f3..f2dc207c6 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateQuarterAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateQuarterAlfrescoSolrTrackerIT.java @@ -33,7 +33,7 @@ import java.util.Properties; @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedDateQuarterAlfrescoSolrTrackerTest extends DistributedDateAbstractSolrTrackerTest +public class DistributedDateQuarterAlfrescoSolrTrackerIT extends DistributedDateAbstractSolrTrackerIT { @BeforeClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateSplitYearAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateSplitYearAlfrescoSolrTrackerIT.java similarity index 93% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateSplitYearAlfrescoSolrTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateSplitYearAlfrescoSolrTrackerIT.java index faa7618c0..ff488c138 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateSplitYearAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateSplitYearAlfrescoSolrTrackerIT.java @@ -33,12 +33,12 @@ import java.util.Properties; @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedDateSplitYearAlfrescoSolrTrackerTest extends DistributedDateAbstractSolrTrackerTest +public class DistributedDateSplitYearAlfrescoSolrTrackerIT extends DistributedDateAbstractSolrTrackerIT { @BeforeClass private static void initData() throws Throwable { - initSolrServers(3, "DistributedDateSplitYearAlfrescoSolrTrackerTest", getShardMethod()); + initSolrServers(3, "DistributedDateSplitYearAlfrescoSolrTrackerIT", getShardMethod()); } @AfterClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java similarity index 96% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java index 8d8655728..f683c0ce1 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java @@ -18,7 +18,7 @@ */ package org.alfresco.solr.tracker; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; @@ -53,12 +53,12 @@ import static org.alfresco.solr.AlfrescoSolrUtils.list; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedDbidRangeAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest +public class DistributedDbidRangeAlfrescoSolrTrackerIT extends AbstractAlfrescoDistributedIT { @BeforeClass private static void initData() throws Throwable { - initSolrServers(2, "DistributedDbidRangeAlfrescoSolrTrackerTest", getShardMethod()); + initSolrServers(2, "DistributedDbidRangeAlfrescoSolrTrackerIT", getShardMethod()); } @AfterClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java index cf6ca9dbd..07c07f0b7 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java @@ -18,7 +18,7 @@ */ package org.alfresco.solr.tracker; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; @@ -56,12 +56,12 @@ import static org.alfresco.solr.AlfrescoSolrUtils.list; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedExpandDbidRangeAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest +public class DistributedExpandDbidRangeAlfrescoSolrTrackerIT extends AbstractAlfrescoDistributedIT { @BeforeClass private static void initData() throws Throwable { - initSolrServers(2, "DistributedExpandDbidRangeAlfrescoSolrTrackerTest", getShardMethod()); + initSolrServers(2, "DistributedExpandDbidRangeAlfrescoSolrTrackerIT", getShardMethod()); } @AfterClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardIdWithStaticPropertyRouterTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardIdWithStaticPropertyRouterIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardIdWithStaticPropertyRouterTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardIdWithStaticPropertyRouterIT.java index 115d81029..7ccf1792e 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardIdWithStaticPropertyRouterTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardIdWithStaticPropertyRouterIT.java @@ -21,7 +21,7 @@ package org.alfresco.solr.tracker; import java.util.Properties; import org.alfresco.model.ContentModel; import org.alfresco.repo.index.shard.ShardMethodEnum; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; @@ -53,7 +53,7 @@ import static org.carrot2.shaded.guava.common.collect.ImmutableList.of; @SolrTestCaseJ4.SuppressSSL @SolrTestCaseJ4.SuppressObjectReleaseTracker (bugUrl = "RAMDirectory") @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedExplicitShardIdWithStaticPropertyRouterTest extends AbstractAlfrescoDistributedTest +public class DistributedExplicitShardIdWithStaticPropertyRouterIT extends AbstractAlfrescoDistributedIT { private static long MAX_WAIT_TIME = 80000; private final int timeout = 100000; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardRoutingTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardRoutingTrackerIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardRoutingTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardRoutingTrackerIT.java index b10bdc275..bd7c5f325 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardRoutingTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardRoutingTrackerIT.java @@ -20,7 +20,7 @@ package org.alfresco.solr.tracker; import org.alfresco.model.ContentModel; import org.alfresco.repo.index.shard.ShardMethodEnum; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; @@ -62,12 +62,12 @@ import static org.alfresco.solr.tracker.DocRouterFactory.SHARD_KEY_KEY; @SolrTestCaseJ4.SuppressSSL @SolrTestCaseJ4.SuppressObjectReleaseTracker (bugUrl = "RAMDirectory") @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedExplicitShardRoutingTrackerTest extends AbstractAlfrescoDistributedTest +public class DistributedExplicitShardRoutingTrackerIT extends AbstractAlfrescoDistributedIT { @BeforeClass private static void initData() throws Throwable { - initSolrServers(3, "DistributedExplicitShardRoutingTrackerTest", getProperties()); + initSolrServers(3, "DistributedExplicitShardRoutingTrackerIT", getProperties()); } @AfterClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedPropertyBasedAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedPropertyBasedAlfrescoSolrTrackerIT.java similarity index 97% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedPropertyBasedAlfrescoSolrTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedPropertyBasedAlfrescoSolrTrackerIT.java index ac01034b2..1e7c8b8f4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedPropertyBasedAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedPropertyBasedAlfrescoSolrTrackerIT.java @@ -41,7 +41,7 @@ import com.carrotsearch.randomizedtesting.RandomizedContext; import org.alfresco.model.ContentModel; import org.alfresco.repo.index.shard.ShardMethodEnum; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; @@ -66,7 +66,7 @@ import org.junit.Test; @SolrTestCaseJ4.SuppressSSL @SolrTestCaseJ4.SuppressObjectReleaseTracker (bugUrl = "RAMDirectory") @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedPropertyBasedAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest +public class DistributedPropertyBasedAlfrescoSolrTrackerIT extends AbstractAlfrescoDistributedIT { private static final String[] DOMAINS = {"alfresco.com", "king.com", "gmail.com", "yahoo.com", "cookie.es"}; private static final Map domainsCount = new HashMap<>(); @@ -78,7 +78,7 @@ public class DistributedPropertyBasedAlfrescoSolrTrackerTest extends AbstractAlf { domainsCount.put(domain,0); } - initSolrServers(4, "DistributedPropertyBasedAlfrescoSolrTrackerTest", getProperties()); + initSolrServers(4, "DistributedPropertyBasedAlfrescoSolrTrackerIT", getProperties()); } @AfterClass diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDRouterTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDRouterIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDRouterTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDRouterIT.java index dd9a04333..0800301f8 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDRouterTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDRouterIT.java @@ -41,7 +41,7 @@ import java.util.Map; import java.util.Properties; @RunWith(MockitoJUnitRunner.class) -public class ExplicitIDRouterTest +public class ExplicitIDRouterIT { private DocRouter router; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDWithLRISRouterTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDWithLRISRouterIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDWithLRISRouterTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDWithLRISRouterIT.java index 2cbdd138c..0cebbbac3 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDWithLRISRouterTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ExplicitIDWithLRISRouterIT.java @@ -42,7 +42,7 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) -public class ExplicitIDWithLRISRouterTest +public class ExplicitIDWithLRISRouterIT { private DocRouter router; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ModelTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ModelTrackerIT.java similarity index 96% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ModelTrackerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ModelTrackerIT.java index 71ee93280..b0a64fe97 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ModelTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ModelTrackerIT.java @@ -56,7 +56,7 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ModelTrackerTest +public class ModelTrackerIT { private ModelTracker modelTracker; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/PropertyRouterTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/PropertyRouterIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/PropertyRouterTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/PropertyRouterIT.java index 67fbdd076..ac233df7a 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/PropertyRouterTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/PropertyRouterIT.java @@ -42,7 +42,7 @@ import java.util.HashMap; import java.util.Map; @RunWith(MockitoJUnitRunner.class) -public class PropertyRouterTest +public class PropertyRouterIT { private PropertyRouter router; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java index ce8889489..4d0314e3c 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java @@ -21,7 +21,7 @@ package org.alfresco.solr.transformer; import org.alfresco.model.ContentModel; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; @@ -58,14 +58,14 @@ import static org.alfresco.solr.AlfrescoSolrUtils.list; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class CachedDocTransformerTest extends AbstractAlfrescoDistributedTest +public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT { public static final String ALFRESCO_JSON = "{\"locales\":[\"en\"], \"templates\": [{\"name\":\"t1\", \"template\":\"%cm:content\"}]}"; @BeforeClass private static void initData() throws Throwable { - initSolrServers(1, "CachedDocTransformerTest", null); + initSolrServers(1, "CachedDocTransformerIT", null); populateAlfrescoData(); } diff --git a/search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterTest.java b/search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterTest.java rename to search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java index be21a6d53..c936e639d 100644 --- a/search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterTest.java +++ b/search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java @@ -56,7 +56,7 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.RegExp; import org.junit.Test; -public class MinHashFilterTest extends BaseTokenStreamTestCase +public class MinHashFilterIT extends BaseTokenStreamTestCase { @Test public void testIntHash() { diff --git a/search-services/alfresco-search/src/test/java/org/apache/solr/handler/component/AlfrescoLukeRequestHandlerTest.java b/search-services/alfresco-search/src/test/java/org/apache/solr/handler/component/AlfrescoLukeRequestHandlerIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/apache/solr/handler/component/AlfrescoLukeRequestHandlerTest.java rename to search-services/alfresco-search/src/test/java/org/apache/solr/handler/component/AlfrescoLukeRequestHandlerIT.java index 7236adddb..9c2a2dfe4 100644 --- a/search-services/alfresco-search/src/test/java/org/apache/solr/handler/component/AlfrescoLukeRequestHandlerTest.java +++ b/search-services/alfresco-search/src/test/java/org/apache/solr/handler/component/AlfrescoLukeRequestHandlerIT.java @@ -43,7 +43,7 @@ import org.mockito.Mock; /** * Unit tests for the {@link AlfrescoLukeRequestHandler}. */ -public class AlfrescoLukeRequestHandlerTest +public class AlfrescoLukeRequestHandlerIT { /** The reference to text for a search term. */ private static final BytesRef TERM_TEXT = new BytesRef("TermText"); From 1795cf0f13e42127fb5924a52c45bf12f8607180 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 1 Nov 2019 11:12:09 +0100 Subject: [PATCH 43/76] [contentStoreReplication] code refactoring. Alfresco header added alongside ASF header. --- .../solr/content/SolrContentStore.java | 10 +- .../solr/handler/AlfrescoIndexFetcher.java | 2704 +++++++++++++++++ .../handler/AlfrescoReplicationHandler.java | 2369 +++++++++++++++ .../solr/handler/OldBackupDirectory.java | 113 +- .../alfresco/solr/handler/SnapShooter.java | 515 ++-- .../test-files/master/conf/solrconfig.xml | 2 +- .../test-files/slave/conf/solrconfig.xml | 2 +- 7 files changed, 5463 insertions(+), 252 deletions(-) create mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java create mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java index 635898b2c..50a9cf662 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java @@ -24,7 +24,7 @@ import org.alfresco.service.cmr.repository.ContentWriter; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.config.ConfigUtil; -import org.alfresco.solr.handler.ReplicationHandler; +import org.alfresco.solr.handler.AlfrescoReplicationHandler; import org.apache.commons.io.FileUtils; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrInputDocument; @@ -326,8 +326,8 @@ public final class SolrContentStore implements Closeable, AccessMode changes.adds.stream() .map(relativePath -> root + relativePath) .map(File::new) - .map(file -> new ReplicationHandler.FileInfo(file, file.getAbsolutePath().replace(root, ""))) - .map(ReplicationHandler.FileInfo::getAsMap) + .map(file -> new AlfrescoReplicationHandler.FileInfo(file, file.getAbsolutePath().replace(root, ""))) + .map(AlfrescoReplicationHandler.FileInfo::getAsMap) .collect(toList())); } @@ -414,8 +414,8 @@ public final class SolrContentStore implements Closeable, AccessMode return Files.walk(Paths.get(root)) .map(Path::toFile) .filter(onlyDatafiles) - .map(file -> new ReplicationHandler.FileInfo(file, file.getAbsolutePath().replace(root, ""))) - .map(ReplicationHandler.FileInfo::getAsMap) + .map(file -> new AlfrescoReplicationHandler.FileInfo(file, file.getAbsolutePath().replace(root, ""))) + .map(AlfrescoReplicationHandler.FileInfo::getAsMap) .collect(toList()); } catch (Exception e) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java new file mode 100644 index 000000000..b9adc4298 --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -0,0 +1,2704 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modification copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see + * + */ +package org.alfresco.solr.handler; + +import static java.util.List.of; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.ALIAS; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CHECKSUM; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_CONTENT_STORE_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_DETAILS; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE_LIST; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_INDEX_VERSION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMMAND; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMPRESSION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILE_SHORT; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILE_LIST; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_VERSION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.EXTERNAL; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE_STREAM; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FileInfo; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.GENERATION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.INTERNAL; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.MASTER_URL; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.NO_INDEX_REPLICATION_REQUIRED; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.OFFSET; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.SIZE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.getCheckSum; +import static org.apache.solr.common.params.CommonParams.JAVABIN; +import static org.apache.solr.common.params.CommonParams.NAME; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import org.alfresco.solr.content.SolrContentStore; +import org.apache.http.client.HttpClient; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.FastInputStream; +import org.apache.solr.common.util.IOUtils; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SuppressForbidden; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.DirectoryFactory.DirContext; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.SnapShooter; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.CdcrUpdateLog; +import org.apache.solr.update.CommitUpdateCommand; +import org.apache.solr.update.UpdateLog; +import org.apache.solr.update.VersionInfo; +import org.apache.solr.util.DefaultSolrThreadFactory; +import org.apache.solr.util.FileUtils; +import org.apache.solr.util.PropertiesOutputStream; +import org.apache.solr.util.RTimer; +import org.apache.solr.util.RefCounted; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.lang.invoke.MethodHandles; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.zip.Adler32; +import java.util.zip.Checksum; +import java.util.zip.InflaterInputStream; + +/** + *

Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the + * master.

+ * + */ +class AlfrescoIndexFetcher +{ + public static final String REPLICATION_PROPERTIES = "replication.properties"; + static final String INDEX_REPLICATED_AT = "indexReplicatedAt"; + static final String TIMES_INDEX_REPLICATED = "timesIndexReplicated"; + static final String CONF_FILES_REPLICATED = "confFilesReplicated"; + static final String CONF_FILES_REPLICATED_AT = "confFilesReplicatedAt"; + static final String TIMES_CONFIG_REPLICATED = "timesConfigReplicated"; + static final String LAST_CYCLE_BYTES_DOWNLOADED = "lastCycleBytesDownloaded"; + static final String TIMES_FAILED = "timesFailed"; + static final String REPLICATION_FAILED_AT = "replicationFailedAt"; + static final String PREVIOUS_CYCLE_TIME_TAKEN = "previousCycleTimeInSeconds"; + static final String INDEX_REPLICATED_AT_LIST = "indexReplicatedAtList"; + static final String REPLICATION_FAILED_AT_LIST = "replicationFailedAtList"; + private static final int _100K = 100000; + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final int CONTENT_STORE_PARTITION_SIZE = 50; + private static final String INTERRUPT_RESPONSE_MESSAGE = "Interrupted while waiting for modify lock"; + private static final int MAX_RETRIES = 5; + private static final int NO_CONTENT = 1; + private static final int ERR = 2; + private final String masterUrl; + private final AlfrescoReplicationHandler alfrescoReplicationHandler; + private final SolrContentStore contentStore; + private final SolrCore solrCore; + private final HttpClient myHttpClient; + private final Map confFileInfoCache = new HashMap<>(); + private volatile Date replicationStartTimeStamp; + private RTimer replicationTimer; + private volatile List> filesToDownload; + private volatile List> confFilesToDownload; + private volatile List> tlogFilesToDownload; + private volatile List> contentStoreFilesToDownload; + private volatile List> contentStoreFilesToDelete; + private volatile List> filesDownloaded; + private volatile List> confFilesDownloaded; + private volatile List> tlogFilesDownloaded; + private volatile List> contentStoreFilesDownloaded; + private volatile Map currentFile; + private volatile DirectoryFileFetcher dirFileFetcher; + private volatile LocalFsFileFetcher localFileFetcher; + private volatile ContentStoreFetcher contentStoreFileFetcher; + private volatile ExecutorService fsyncService; + private volatile boolean stop = false; + private boolean useInternalCompression; + private boolean useExternalCompression; + private boolean fullContentStoreReplication = false; + private volatile Exception fsyncException; + + AlfrescoIndexFetcher(final NamedList initArgs, final AlfrescoReplicationHandler handler, final SolrCore sc, SolrContentStore contentStore) + { + this.contentStore = contentStore; + solrCore = sc; + String masterUrl = (String) initArgs.get(MASTER_URL); + if (masterUrl == null) + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "'masterUrl' is required for a slave"); + if (masterUrl.endsWith(AlfrescoReplicationHandler.PATH)) + { + masterUrl = masterUrl.substring(0, masterUrl.length() - 12); + LOG.warn("'masterUrl' must be specified without the " + AlfrescoReplicationHandler.PATH + " suffix"); + } + this.masterUrl = masterUrl; + + this.alfrescoReplicationHandler = handler; + String compress = (String) initArgs.get(COMPRESSION); + useInternalCompression = INTERNAL.equals(compress); + useExternalCompression = EXTERNAL.equals(compress); + String connTimeout = (String) initArgs.get(HttpClientUtil.PROP_CONNECTION_TIMEOUT); + // allow a master override for tests - you specify this in /replication slave section of solrconfig and some + // test don't want to define this + String readTimeout = System.getProperty("solr.indexfetcher.sotimeout", "-1"); + if (readTimeout.equals("-1")) + { + readTimeout = (String) initArgs.get(HttpClientUtil.PROP_SO_TIMEOUT); + } + String httpBasicAuthUser = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_USER); + String httpBasicAuthPassword = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_PASS); + myHttpClient = createHttpClient(solrCore, connTimeout, readTimeout, httpBasicAuthUser, httpBasicAuthPassword, + useExternalCompression); + } + + private static HttpClient createHttpClient(SolrCore core, String connTimeout, String readTimeout, String httpBasicAuthUser, String httpBasicAuthPassword, boolean useCompression) + { + final ModifiableSolrParams httpClientParams = new ModifiableSolrParams(); + httpClientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connTimeout != null ? connTimeout : "5000"); + httpClientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, readTimeout != null ? readTimeout : "20000"); + httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_USER, httpBasicAuthUser); + httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword); + httpClientParams.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, useCompression); + + return HttpClientUtil + .createClient(httpClientParams, core.getCoreContainer().getUpdateShardHandler().getConnectionManager()); + } + + private static boolean filesToAlwaysDownloadIfNoChecksums(String filename, long size, CompareResult compareResult) + { + // without checksums to compare, we always download .si, .liv, segments_N, + // and any very small files + return !compareResult.checkSummed && (filename.endsWith(".si") || filename.endsWith(".liv") || filename + .startsWith("segments_") || size < _100K); + } + + /** + * Check if the same version of the file to download is already in the contentstore. + * + * @param contentStoreDirectoryRoot the content store top directory. + * @param filename the filename. + * @param length the file length. + * @param checksum the file chechsum. + * @return true if the same file already exists in the contentstore. + */ + private static boolean compareContentStoreFiles(File contentStoreDirectoryRoot, String filename, long length, long checksum) + { + File f = new File(contentStoreDirectoryRoot.getAbsolutePath() + "/" + filename); + if (f.length() != length) + { + return true; + } + + Checksum localCheckSum = new Adler32(); + + if (getCheckSum(localCheckSum, f) != checksum) + { + return true; + } + + LOG.debug("{} already in contentstore", filename); + return false; + } + + private static CompareResult compareFile(Directory indexDir, String filename, Long backupIndexFileLen, Long backupIndexFileChecksum) + { + CompareResult compareResult = new CompareResult(); + try + { + try (final IndexInput indexInput = indexDir.openInput(filename, IOContext.READONCE)) + { + long indexFileLen = indexInput.length(); + long indexFileChecksum = 0; + + if (backupIndexFileChecksum != null) + { + try + { + indexFileChecksum = CodecUtil.retrieveChecksum(indexInput); + compareResult.checkSummed = true; + } + catch (Exception e) + { + LOG.warn("Could not retrieve checksum from file.", e); + } + } + + if (!compareResult.checkSummed) + { + // we don't have checksums to compare + + if (indexFileLen == backupIndexFileLen) + { + compareResult.equal = true; + return compareResult; + } + else + { + LOG.info("File {} did not match. expected length is {} and actual length is {}", filename, + backupIndexFileLen, indexFileLen); + compareResult.equal = false; + return compareResult; + } + } + + // we have checksums to compare + + if (indexFileLen == backupIndexFileLen && indexFileChecksum == backupIndexFileChecksum) + { + compareResult.equal = true; + return compareResult; + } + else + { + LOG.warn("File {} did not match. expected checksum is {} and actual is checksum {}. " + + "expected length is {} and actual length is {}", filename, backupIndexFileChecksum, + indexFileChecksum, backupIndexFileLen, indexFileLen); + compareResult.equal = false; + return compareResult; + } + } + } + catch (NoSuchFileException | FileNotFoundException e) + { + compareResult.equal = false; + return compareResult; + } + catch (IOException e) + { + LOG.error("Could not read file " + filename + ". Downloading it again", e); + compareResult.equal = false; + return compareResult; + } + } + + /** + * Returns true if the file exists (can be opened), false + * if it cannot be opened, and (unlike Java's + * File.exists) throws IOException if there's some + * unexpected error. + */ + private static boolean slowFileExists(Directory dir, String fileName) throws IOException + { + try + { + dir.openInput(fileName, IOContext.DEFAULT).close(); + return true; + } + catch (NoSuchFileException | FileNotFoundException e) + { + return false; + } + } + + static boolean delTree(File dir) + { + try + { + org.apache.lucene.util.IOUtils.rm(dir.toPath()); + return true; + } + catch (IOException e) + { + LOG.warn("Unable to delete directory : " + dir, e); + return false; + } + } + + /** + * Gets the latest commit version and generation from the master + */ + private NamedList getLatestVersion() throws IOException + { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(COMMAND, CMD_INDEX_VERSION); + params.set(CommonParams.WT, JAVABIN); + params.set(CommonParams.QT, AlfrescoReplicationHandler.PATH); + QueryRequest req = new QueryRequest(params); + + // TODO modify to use shardhandler + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) + { + client.setSoTimeout(60000); + client.setConnectionTimeout(15000); + + return client.request(req); + } + catch (SolrServerException e) + { + throw new SolrException(ErrorCode.SERVER_ERROR, e.getMessage(), e); + } + } + + /** + * Fetches the list of files in a given index commit point and updates internal list of files to download. + */ + @SuppressWarnings("unchecked") + private void fetchFileList(long indexGeneration, long contentStoreGeneration) throws IOException + { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(COMMAND, CMD_GET_FILE_LIST); + params.set(GENERATION, String.valueOf(indexGeneration)); + params.set(CONTENT_STORE_VERSION, String.valueOf(contentStoreGeneration)); + params.set(CommonParams.WT, JAVABIN); + params.set(CommonParams.QT, AlfrescoReplicationHandler.PATH); + QueryRequest req = new QueryRequest(params); + + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) + { + client.setSoTimeout(60000); + client.setConnectionTimeout(15000); + NamedList response = client.request(req); + + List> files = (List>) response.get(CMD_GET_FILE_LIST); + if (files != null) + { + filesToDownload = Collections.synchronizedList(files); + } + else + { + filesToDownload = Collections.emptyList(); + LOG.info("No files to download for index generation: " + indexGeneration); + } + + files = (List>) response.get(CONF_FILES); + if (files != null) + { + confFilesToDownload = Collections.synchronizedList(files); + } + + files = (List>) response.get(TLOG_FILES); + if (files != null) + { + tlogFilesToDownload = Collections.synchronizedList(files); + } + Map>> contentStoreMap = (Map>>) response + .get(CONTENT_STORE_FILES); + + if (contentStoreMap != null) + { + contentStoreFilesToDownload = Collections.synchronizedList(contentStoreMap.get(SolrContentStore.ADDS)); + contentStoreFilesToDelete = Collections.synchronizedList(contentStoreMap.get(SolrContentStore.DELETES)); + + List> infomap = contentStoreMap.get(SolrContentStore.INFO); + fullContentStoreReplication = of(infomap).stream().flatMap(List::stream) + .map(e -> e.get(SolrContentStore.FULL_REPLICATION)) + .map(e -> Boolean.class.isInstance(e) ? (Boolean) e : false).findFirst().orElse(false); + + } + + } + catch (SolrServerException e) + { + throw new IOException(e); + } + } + + IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean replicateContentStore) + throws IOException, InterruptedException + { + return fetchLatestIndex(forceReplication, false, replicateContentStore); + } + + /** + * This command downloads all the necessary files from master to install a index commit point. Only changed files are + * downloaded. It also downloads the conf files (if they are modified). + * + * @param forceReplication force a replication in all cases + * @param forceCoreReload force a core reload in all cases + * @return true on success, false if slave is already in sync + * @throws IOException if an exception occurs + */ + private IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreReload, boolean replicateContentStore) + throws IOException, InterruptedException + { + + boolean cleanupDone = false; + boolean successfulInstall = false; + markReplicationStart(); + Directory tmpIndexDir = null; + String tmpIndex; + Directory indexDir = null; + String indexDirPath; + boolean deleteTmpIdxDir = true; + File tmpTlogDir = null; + + if (!solrCore.getSolrCoreState().getLastReplicateIndexSuccess()) + { + // if the last replication was not a success, we force a full replication + // when we are a bit more confident we may want to try a partial replication + // if the error is connection related or something, but we have to be careful + forceReplication = true; + } + + try + { + //get the current 'replicateable' index version in the master + NamedList response; + try + { + response = getLatestVersion(); + } + catch (Exception e) + { + final String errorMsg = e.toString(); + if (!Strings.isNullOrEmpty(errorMsg) && errorMsg.contains(INTERRUPT_RESPONSE_MESSAGE)) + { + LOG.warn("Master at: " + masterUrl + + " is not available. Index fetch failed by interrupt. Exception: " + errorMsg); + return new IndexFetchResult(IndexFetchResult.FAILED_BY_INTERRUPT_MESSAGE, false, e); + } + else + { + LOG.warn("Master at: " + masterUrl + " is not available. Index fetch failed by exception: " + + errorMsg); + return new IndexFetchResult(IndexFetchResult.FAILED_BY_EXCEPTION_MESSAGE, false, e); + } + } + + long latestVersion = (Long) response.get(CMD_INDEX_VERSION); + long masterContentStoreVersion = (Long) response.get(CONTENT_STORE_VERSION); + long latestGeneration = (Long) response.get(GENERATION); + + // The following session should make sure that if replication is happening in more cores at the same time, + // the contentStore replication is done only once. + long slaveContentStoreVersion = replicateContentStore ? + contentStore.getLastCommittedVersion() : + SolrContentStore.NO_CONTENT_STORE_REPLICATION_REQUIRED; + boolean contentStoreReplicationNeeded = + replicateContentStore && (masterContentStoreVersion != slaveContentStoreVersion); + boolean indexReplicationNeeded = true; + + LOG.info("Master's generation: " + latestGeneration); + LOG.info("Master's version: " + latestVersion); + + IndexCommit commit = solrCore.getDeletionPolicy().getLatestCommit(); + if (commit == null) + { + // Presumably the IndexWriter hasn't been opened yet, and hence the deletion policy hasn't been updated with commit points + RefCounted searcherRefCounted = null; + try + { + searcherRefCounted = solrCore.getNewestSearcher(false); + if (searcherRefCounted == null) + { + LOG.warn("No open searcher found - fetch aborted"); + return IndexFetchResult.NO_INDEX_COMMIT_EXIST; + } + commit = searcherRefCounted.get().getIndexReader().getIndexCommit(); + } + finally + { + if (searcherRefCounted != null) + searcherRefCounted.decref(); + } + } + + LOG.info("Slave's generation: " + commit.getGeneration()); + LOG.info("Slave's version: " + IndexDeletionPolicyWrapper.getCommitTimestamp(commit)); + + if (latestVersion == 0L) + { + if (forceReplication && commit.getGeneration() != 0) + { + // since we won't get the files for an empty index, + // we just clear ours and commit + RefCounted iw = solrCore.getUpdateHandler().getSolrCoreState() + .getIndexWriter(solrCore); + try + { + iw.get().deleteAll(); + } + finally + { + iw.decref(); + } + SolrQueryRequest req = new LocalSolrQueryRequest(solrCore, new ModifiableSolrParams()); + solrCore.getUpdateHandler().commit(new CommitUpdateCommand(req, false)); + } + + //there is nothing to be replicated + successfulInstall = true; + return IndexFetchResult.MASTER_VERSION_ZERO; + } + + if (!forceReplication && IndexDeletionPolicyWrapper.getCommitTimestamp(commit) == latestVersion) + { + //master and slave are already in sync just return + LOG.info("Slave index in sync with master."); + successfulInstall = true; + indexReplicationNeeded = false; + } + + if (!indexReplicationNeeded && !contentStoreReplicationNeeded) + { + return IndexFetchResult.ALREADY_IN_SYNC; + } + + if (indexReplicationNeeded) + LOG.info("Starting index replication process"); + + if (contentStoreReplicationNeeded) + LOG.info("Starting content store replication process"); + + if (masterContentStoreVersion < slaveContentStoreVersion) + { + LOG.error("slave content store version is not valid. Full content store replication required"); + slaveContentStoreVersion = SolrContentStore.NO_VERSION_AVAILABLE; + } + + // get the list of files first + fetchFileList(indexReplicationNeeded ? latestGeneration : NO_INDEX_REPLICATION_REQUIRED, + slaveContentStoreVersion); + // this can happen if the commit point is deleted before we fetch the file list. + if (filesToDownload.isEmpty() && indexReplicationNeeded) + { + return IndexFetchResult.PEER_INDEX_COMMIT_DELETED; + } + + if (indexReplicationNeeded) + { + + LOG.info("Number of files in latest index in master: " + filesToDownload.size()); + if (tlogFilesToDownload != null) + { + LOG.info("Number of tlog files in master: " + tlogFilesToDownload.size()); + } + } + + // Create the sync service + fsyncService = ExecutorUtil.newMDCAwareSingleThreadExecutor(new DefaultSolrThreadFactory("fsyncService")); + // use a synchronized list because the list is read by other threads (to show details) + filesDownloaded = Collections.synchronizedList(new ArrayList<>()); + + // if the generation of master is older than that of the slave , it means they are not compatible to be copied + // then a new index directory to be created and all the files need to be copied + boolean isFullCopyNeeded = + indexReplicationNeeded && (IndexDeletionPolicyWrapper.getCommitTimestamp(commit) >= latestVersion + || commit.getGeneration() >= latestGeneration || forceReplication); + + String timestamp = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date()); + String tmpIdxDirName = "index." + timestamp; + tmpIndex = solrCore.getDataDir() + tmpIdxDirName; + + tmpIndexDir = solrCore.getDirectoryFactory() + .get(tmpIndex, DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); + + // tmp dir for tlog files + if (tlogFilesToDownload != null) + { + tmpTlogDir = new File(solrCore.getUpdateHandler().getUpdateLog().getLogDir(), "tlog." + timestamp); + } + + // cindex dir... + indexDirPath = solrCore.getIndexDir(); + indexDir = solrCore.getDirectoryFactory() + .get(indexDirPath, DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); + + try + { + + if (indexReplicationNeeded) + { + + //We will compare all the index files from the master vs the index files on disk to see if there is a mismatch + //in the metadata. If there is a mismatch for the same index file then we download the entire index again. + if (!isFullCopyNeeded && isIndexStale(indexDir)) + { + isFullCopyNeeded = true; + } + + if (!isFullCopyNeeded) + { + // a searcher might be using some flushed but not committed segments + // because of soft commits (which open a searcher on IW's data) + // so we need to close the existing searcher on the last commit + // and wait until we are able to clean up all unused lucene files + if (solrCore.getCoreContainer().isZooKeeperAware()) + { + solrCore.closeSearcher(); + } + + // rollback and reopen index writer and wait until all unused files + // are successfully deleted + solrCore.getUpdateHandler().newIndexWriter(true); + RefCounted writer = solrCore.getUpdateHandler().getSolrCoreState() + .getIndexWriter(null); + try + { + IndexWriter indexWriter = writer.get(); + int c = 0; + indexWriter.deleteUnusedFiles(); + while (hasUnusedFiles(indexDir, commit)) + { + indexWriter.deleteUnusedFiles(); + LOG.info("Sleeping for 1000ms to wait for unused lucene index files to be delete-able"); + Thread.sleep(1000); + c++; + if (c >= 30) + { + LOG.warn( + "IndexFetcher unable to cleanup unused lucene index files so we must do a full copy instead"); + isFullCopyNeeded = true; + break; + } + } + if (c > 0) + { + LOG.info("IndexFetcher slept for " + (c * 1000) + + "ms for unused lucene index files to be delete-able"); + } + } + finally + { + writer.decref(); + } + } + } + + boolean reloadCore = false; + + try + { + // we have to be careful and do this after we know isFullCopyNeeded won't be flipped + if (!isFullCopyNeeded) + { + solrCore.getUpdateHandler().getSolrCoreState().closeIndexWriter(solrCore, true); + } + + LOG.info("Starting download (fullCopy={}) to {}", isFullCopyNeeded, tmpIndexDir); + successfulInstall = false; + + long bytesDownloaded = 0; + + if (indexReplicationNeeded) + { + downloadIndexFiles(isFullCopyNeeded, indexDir, tmpIndexDir, latestGeneration); + + if (tlogFilesToDownload != null) + { + bytesDownloaded += downloadTlogFiles(tmpTlogDir, latestGeneration); + reloadCore = true; // reload update log + } + } + + if (contentStoreReplicationNeeded) + { + + if (contentStoreFilesToDownload != null) + { + bytesDownloaded += downloadContentStoreFiles(contentStore.getRootLocation()); + } + + if (contentStoreFilesToDelete != null) + { + deleteContentStoreFiles(contentStore.getRootLocation(), contentStoreFilesToDelete); + } + + if (fullContentStoreReplication) + { + deleteUnnecessaryContentStoreFiles(contentStore.getRootLocation()); + } + + contentStore.setLastCommittedVersion(masterContentStoreVersion); + } + + final long timeTakenSeconds = getReplicationTimeElapsed(); + final Long bytesDownloadedPerSecond = (timeTakenSeconds != 0 ? + new Long(bytesDownloaded / timeTakenSeconds) : + null); + LOG.info("Total time taken for download (fullCopy={},bytesDownloaded={}) : {} secs ({} bytes/sec)", + new Object[] { isFullCopyNeeded, bytesDownloaded, timeTakenSeconds, + bytesDownloadedPerSecond }); + + if (indexReplicationNeeded) + { + + Collection> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload); + if (!modifiedConfFiles.isEmpty()) + { + reloadCore = true; + downloadConfFiles(confFilesToDownload, latestGeneration); + if (isFullCopyNeeded) + { + successfulInstall = solrCore.modifyIndexProps(tmpIdxDirName); + if (successfulInstall) + deleteTmpIdxDir = false; + } + else + { + successfulInstall = moveIndexFiles(tmpIndexDir, indexDir); + } + if (tlogFilesToDownload != null) + { + // move tlog files and refresh ulog only if we successfully installed a new index + successfulInstall &= moveTlogFiles(tmpTlogDir); + } + if (successfulInstall) + { + if (isFullCopyNeeded) + { + // let the system know we are changing dir's and the old one + // may be closed + if (indexDir != null) + { + solrCore.getDirectoryFactory().doneWithDirectory(indexDir); + // Cleanup all index files not associated with any *named* snapshot. + solrCore.deleteNonSnapshotIndexFiles(indexDirPath); + } + } + + LOG.info("Configuration files are modified, core will be reloaded"); + logReplicationTimeAndConfFiles(modifiedConfFiles, + successfulInstall);// write to a file time of replication and + // conf files. + } + } + else + { + terminateAndWaitFsyncService(); + if (isFullCopyNeeded) + { + successfulInstall = solrCore.modifyIndexProps(tmpIdxDirName); + if (successfulInstall) + deleteTmpIdxDir = false; + } + else + { + successfulInstall = moveIndexFiles(tmpIndexDir, indexDir); + } + if (tlogFilesToDownload != null) + { + // move tlog files and refresh ulog only if we successfully installed a new index + successfulInstall &= moveTlogFiles(tmpTlogDir); + } + if (successfulInstall) + { + logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall); + } + } + } + } + finally + { + if (!isFullCopyNeeded && indexReplicationNeeded) + { + solrCore.getUpdateHandler().getSolrCoreState().openIndexWriter(solrCore); + } + } + + if (indexReplicationNeeded) + { + + // we must reload the core after we open the IW back up + if (successfulInstall && (reloadCore || forceCoreReload)) + { + LOG.info("Reloading SolrCore {}", solrCore.getName()); + reloadCore(); + } + + if (successfulInstall) + { + if (isFullCopyNeeded) + { + // let the system know we are changing dir's and the old one + // may be closed + if (indexDir != null) + { + LOG.info("removing old index directory " + indexDir); + solrCore.getDirectoryFactory().doneWithDirectory(indexDir); + solrCore.getDirectoryFactory().remove(indexDir); + } + } + if (isFullCopyNeeded) + { + solrCore.getUpdateHandler().newIndexWriter(isFullCopyNeeded); + } + + openNewSearcherAndUpdateCommitPoint(); + } + + if (!isFullCopyNeeded && !forceReplication && !successfulInstall) + { + cleanup(solrCore, tmpIndexDir, indexDir, deleteTmpIdxDir, tmpTlogDir, successfulInstall); + cleanupDone = true; + // we try with a full copy of the index + LOG.warn( + "Replication attempt was not successful - trying a full index replication reloadCore={}", + reloadCore); + successfulInstall = fetchLatestIndex(true, reloadCore).getSuccessful(); + } + } + + markReplicationStop(); + return successfulInstall ? IndexFetchResult.INDEX_FETCH_SUCCESS : IndexFetchResult.INDEX_FETCH_FAILURE; + } + catch (ReplicationHandlerException e) + { + LOG.error("User aborted Replication"); + return new IndexFetchResult(IndexFetchResult.FAILED_BY_EXCEPTION_MESSAGE, false, e); + } + catch (SolrException e) + { + throw e; + } + catch (InterruptedException e) + { + throw new InterruptedException("Index fetch interrupted"); + } + catch (Exception e) + { + throw new SolrException(ErrorCode.SERVER_ERROR, "Index fetch failed : ", e); + } + } + finally + { + if (!cleanupDone) + { + cleanup(solrCore, tmpIndexDir, indexDir, deleteTmpIdxDir, tmpTlogDir, successfulInstall); + } + } + } + + /** + * Download content store required files. + */ + private long downloadContentStoreFiles(String contentStoreDirectory) throws Exception + { + LOG.info("Starting download of content store files from master: " + tlogFilesToDownload); + contentStoreFilesDownloaded = Collections.synchronizedList(new ArrayList<>()); + long bytesDownloaded = 0; + + File tmpContentStoreDirectory = new File(contentStoreDirectory, "contentstore." + getDateAsStr(new Date())); + + LOG.info("content store replication: {} files to downlad", contentStoreFilesToDownload.size()); + + List> contentStoreFilesToDownloadFiltered = contentStoreFilesToDownload.stream() + .filter(file -> compareContentStoreFiles(new File(contentStoreDirectory), (String) file.get(NAME), + (Long) file.get(SIZE), (Long) file.get(CHECKSUM))).collect(Collectors.toList()); + + if (contentStoreFilesToDownloadFiltered.size() != contentStoreFilesToDownload.size()) + { + LOG.warn("content store replication: some of the files are already in sync. {} files to download", + contentStoreFilesToDownloadFiltered.size()); + } + + if (!contentStoreFilesToDownloadFiltered.isEmpty()) + { + for (List> partition : Lists + .partition(contentStoreFilesToDownloadFiltered, CONTENT_STORE_PARTITION_SIZE)) + { + contentStoreFileFetcher = new ContentStoreFetcher(tmpContentStoreDirectory, + AlfrescoReplicationHandler.CONTENT_STORE_FILES, partition); + + contentStoreFileFetcher.fetchContentStore(); + bytesDownloaded += contentStoreFileFetcher.getBytesDownloaded(); + contentStoreFilesDownloaded.addAll(partition); + } + + terminateAndWaitFsyncService(); + copyTmpContentStoreToContentStore(tmpContentStoreDirectory, contentStoreDirectory); + + delTree(tmpContentStoreDirectory); + LOG.info("content store files successfully downloaded"); + } + + return bytesDownloaded; + } + + private void cleanup(final SolrCore core, Directory tmpIndexDir, Directory indexDir, boolean deleteTmpIdxDir, File tmpTlogDir, boolean successfulInstall) + { + try + { + if (!successfulInstall) + { + try + { + logReplicationTimeAndConfFiles(null, successfulInstall); + } + catch (Exception e) + { + // this can happen on shutdown, a fetch may be running in a thread after DirectoryFactory is closed + LOG.warn("Could not log failed replication details", e); + } + } + + if (core.getCoreContainer().isZooKeeperAware()) + { + // we only track replication success in SolrCloud mode + core.getUpdateHandler().getSolrCoreState().setLastReplicateIndexSuccess(successfulInstall); + } + + filesToDownload = filesDownloaded = confFilesDownloaded = confFilesToDownload = tlogFilesToDownload = tlogFilesDownloaded = contentStoreFilesDownloaded = contentStoreFilesToDownload = null; + markReplicationStop(); + dirFileFetcher = null; + localFileFetcher = null; + if (fsyncService != null && !fsyncService.isShutdown()) + { + fsyncService.shutdown(); + } + fsyncService = null; + stop = false; + fsyncException = null; + } + finally + { + // order below is important + try + { + if (tmpIndexDir != null && deleteTmpIdxDir) + { + core.getDirectoryFactory().doneWithDirectory(tmpIndexDir); + core.getDirectoryFactory().remove(tmpIndexDir); + } + } + catch (Exception e) + { + SolrException.log(LOG, e); + } + finally + { + try + { + if (tmpIndexDir != null){ + core.getDirectoryFactory().release(tmpIndexDir); + } + } + catch (Exception e) + { + SolrException.log(LOG, e); + } + try + { + if (indexDir != null) + { + core.getDirectoryFactory().release(indexDir); + } + } + catch (Exception e) + { + SolrException.log(LOG, e); + } + try + { + if (tmpTlogDir != null) + { + delTree(tmpTlogDir); + } + } + catch (Exception e) + { + SolrException.log(LOG, e); + } + } + } + } + + private boolean hasUnusedFiles(Directory indexDir, IndexCommit commit) throws IOException + { + String segmentsFileName = commit.getSegmentsFileName(); + SegmentInfos infos = SegmentInfos.readCommit(indexDir, segmentsFileName); + Set currentFiles = new HashSet<>(infos.files(true)); + String[] allFiles = indexDir.listAll(); + for (String file : allFiles) + { + if (!file.equals(segmentsFileName) && !currentFiles.contains(file) && !file.endsWith(".lock")) + { + LOG.info("Found unused file: " + file); + return true; + } + } + return false; + } + + /** + * terminate the fsync service and wait for all the tasks to complete. If it is already terminated + */ + private void terminateAndWaitFsyncService() throws Exception + { + if (fsyncService.isTerminated()){ + return; + } + fsyncService.shutdown(); + // give a long wait say 1 hr + fsyncService.awaitTermination(3600, TimeUnit.SECONDS); + // if any fsync failed, throw that exception back + Exception fsyncExceptionCopy = fsyncException; + if (fsyncExceptionCopy != null) + { + throw fsyncExceptionCopy; + } + } + + /** + * Helper method to record the last replication's details so that we can show them on the statistics page across + * restarts. + * + * @throws IOException on IO error + */ + @SuppressForbidden(reason = "Need currentTimeMillis for debugging/stats") + private void logReplicationTimeAndConfFiles(Collection> modifiedConfFiles, boolean successfulInstall) + throws IOException + { + List confFiles = new ArrayList<>(); + if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty()){ + for (Map map1 : modifiedConfFiles) + { + confFiles.add((String) map1.get(NAME)); + } + } + + Properties props = alfrescoReplicationHandler.loadReplicationProperties(); + long replicationTime = System.currentTimeMillis(); + long replicationTimeTaken = getReplicationTimeElapsed(); + Directory dir = null; + try + { + dir = solrCore.getDirectoryFactory() + .get(solrCore.getDataDir(), DirContext.META_DATA, solrCore.getSolrConfig().indexConfig.lockType); + + int indexCount = 1, confFilesCount = 1; + if (props.containsKey(TIMES_INDEX_REPLICATED)) + { + indexCount = Integer.parseInt(props.getProperty(TIMES_INDEX_REPLICATED)) + 1; + } + StringBuilder sb = readToStringBuilder(replicationTime, props.getProperty(INDEX_REPLICATED_AT_LIST)); + props.setProperty(INDEX_REPLICATED_AT_LIST, sb.toString()); + props.setProperty(INDEX_REPLICATED_AT, String.valueOf(replicationTime)); + props.setProperty(PREVIOUS_CYCLE_TIME_TAKEN, String.valueOf(replicationTimeTaken)); + props.setProperty(TIMES_INDEX_REPLICATED, String.valueOf(indexCount)); + if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty()) + { + props.setProperty(CONF_FILES_REPLICATED, confFiles.toString()); + props.setProperty(CONF_FILES_REPLICATED_AT, String.valueOf(replicationTime)); + if (props.containsKey(TIMES_CONFIG_REPLICATED)) + { + confFilesCount = Integer.parseInt(props.getProperty(TIMES_CONFIG_REPLICATED)) + 1; + } + props.setProperty(TIMES_CONFIG_REPLICATED, String.valueOf(confFilesCount)); + } + + props.setProperty(LAST_CYCLE_BYTES_DOWNLOADED, String.valueOf(getTotalBytesDownloaded())); + if (!successfulInstall) + { + int numFailures = 1; + if (props.containsKey(TIMES_FAILED)) + { + numFailures = Integer.parseInt(props.getProperty(TIMES_FAILED)) + 1; + } + props.setProperty(TIMES_FAILED, String.valueOf(numFailures)); + props.setProperty(REPLICATION_FAILED_AT, String.valueOf(replicationTime)); + sb = readToStringBuilder(replicationTime, props.getProperty(REPLICATION_FAILED_AT_LIST)); + props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString()); + } + + String tmpFileName = REPLICATION_PROPERTIES + "." + System.nanoTime(); + final IndexOutput out = dir.createOutput(tmpFileName, DirectoryFactory.IOCONTEXT_NO_CACHE); + Writer outFile = new OutputStreamWriter(new PropertiesOutputStream(out), StandardCharsets.UTF_8); + try + { + props.store(outFile, "Replication details"); + dir.sync(Collections.singleton(tmpFileName)); + } + finally + { + IOUtils.closeQuietly(outFile); + } + + solrCore.getDirectoryFactory().renameWithOverwrite(dir, tmpFileName, REPLICATION_PROPERTIES); + } + catch (Exception e) + { + LOG.warn("Exception while updating statistics", e); + } + finally + { + if (dir != null) + { + solrCore.getDirectoryFactory().release(dir); + } + } + } + + long getTotalBytesDownloaded() + { + long bytesDownloaded = 0; + //get size from list of files to download + for (Map file : getFilesDownloaded()) + { + bytesDownloaded += (Long) file.get(SIZE); + } + + //get size from list of conf files to download + for (Map file : getConfFilesDownloaded()) + { + bytesDownloaded += (Long) file.get(SIZE); + } + + //get size from current file being downloaded + Map currentFile = getCurrentFile(); + if (currentFile != null) + { + if (currentFile.containsKey("bytesDownloaded")) + { + bytesDownloaded += (Long) currentFile.get("bytesDownloaded"); + } + } + return bytesDownloaded; + } + + private StringBuilder readToStringBuilder(long replicationTime, String str) + { + StringBuilder sb = new StringBuilder(); + List l = new ArrayList<>(); + if (str != null && str.length() != 0) + { + String[] ss = str.split(","); + Collections.addAll(l, ss); + } + sb.append(replicationTime); + if (!l.isEmpty()) + { + for (int i = 0; i < l.size() || i < 9; i++) + { + if (i == l.size() || i == 9) + { + break; + } + String s = l.get(i); + sb.append(",").append(s); + } + } + return sb; + } + + private void openNewSearcherAndUpdateCommitPoint() throws IOException + { + RefCounted searcher = null; + IndexCommit commitPoint; + // must get the latest solrCore object because the one we have might be closed because of a reload + // todo stop keeping solrCore around + SolrCore core = solrCore.getCoreContainer().getCore(solrCore.getName()); + try + { + Future[] waitSearcher = new Future[1]; + searcher = core.getSearcher(true, true, waitSearcher, true); + if (waitSearcher[0] != null) + { + try + { + waitSearcher[0].get(); + } + catch (InterruptedException | ExecutionException e) + { + SolrException.log(LOG, e); + } + } + commitPoint = searcher.get().getIndexReader().getIndexCommit(); + } + finally + { + if (searcher != null) + { + searcher.decref(); + } + core.close(); + } + + // update the commit point in replication handler + alfrescoReplicationHandler.indexCommitPoint = commitPoint; + + } + + private void reloadCore() + { + final CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + try + { + solrCore.getCoreContainer().reload(solrCore.getName()); + } + catch (Exception e) + { + LOG.error("Could not reload core ", e); + } + finally + { + latch.countDown(); + } + }).start(); + try + { + latch.await(); + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting for core reload to finish", e); + } + } + + private void downloadConfFiles(List> confFilesToDownload, long latestGeneration) + throws Exception + { + LOG.info("Starting download of configuration files from master: " + confFilesToDownload); + confFilesDownloaded = Collections.synchronizedList(new ArrayList<>()); + File tmpconfDir = new File(solrCore.getResourceLoader().getConfigDir(), "conf." + getDateAsStr(new Date())); + try + { + boolean status = tmpconfDir.mkdirs(); + if (!status) + { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to create temporary config folder: " + tmpconfDir.getName()); + } + for (Map file : confFilesToDownload) + { + String saveAs = (String) (file.get(ALIAS) == null ? file.get(NAME) : file.get(ALIAS)); + localFileFetcher = new LocalFsFileFetcher(tmpconfDir, file, saveAs, CONF_FILE_SHORT, latestGeneration); + currentFile = file; + localFileFetcher.fetchFile(); + confFilesDownloaded.add(new HashMap<>(file)); + } + // this is called before copying the files to the original conf dir + // so that if there is an exception avoid corrupting the original files. + terminateAndWaitFsyncService(); + copyTmpConfFiles2Conf(tmpconfDir); + } + finally + { + delTree(tmpconfDir); + } + } + + /** + * Download all the tlog files to the temp tlog directory. + */ + private long downloadTlogFiles(File tmpTlogDir, long latestGeneration) throws Exception + { + LOG.info("Starting download of tlog files from master: " + tlogFilesToDownload); + tlogFilesDownloaded = Collections.synchronizedList(new ArrayList<>()); + long bytesDownloaded = 0; + + boolean status = tmpTlogDir.mkdirs(); + if (!status) + { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to create temporary tlog folder: " + tmpTlogDir.getName()); + } + for (Map file : tlogFilesToDownload) + { + String saveAs = (String) (file.get(ALIAS) == null ? file.get(NAME) : file.get(ALIAS)); + localFileFetcher = new LocalFsFileFetcher(tmpTlogDir, file, saveAs, TLOG_FILE, latestGeneration); + currentFile = file; + localFileFetcher.fetchFile(); + bytesDownloaded += localFileFetcher.getBytesDownloaded(); + tlogFilesDownloaded.add(new HashMap<>(file)); + } + return bytesDownloaded; + } + + /** + * Download the index files. If a new index is needed, download all the files. + * + * @param downloadCompleteIndex is it a fresh index copy + * @param tmpIndexDir the directory to which files need to be downloadeed to + * @param indexDir the indexDir to be merged to + * @param latestGeneration the version number + * @return number of bytes downloaded + */ + private long downloadIndexFiles(boolean downloadCompleteIndex, Directory indexDir, Directory tmpIndexDir, long latestGeneration) + throws Exception + { + if (LOG.isDebugEnabled()) + { + LOG.debug("Download files to dir: " + Arrays.asList(indexDir.listAll())); + } + long bytesDownloaded = 0; + for (Map file : filesToDownload) + { + String filename = (String) file.get(NAME); + long size = (Long) file.get(SIZE); + CompareResult compareResult = compareFile(indexDir, filename, size, (Long) file.get(CHECKSUM)); + boolean alwaysDownload = filesToAlwaysDownloadIfNoChecksums(filename, size, compareResult); + + LOG.debug("Downloading file={} size={} checksum={} alwaysDownload={}", filename, size, file.get(CHECKSUM), + alwaysDownload); + + if (!compareResult.equal || downloadCompleteIndex || alwaysDownload) + { + dirFileFetcher = new DirectoryFileFetcher(tmpIndexDir, file, (String) file.get(NAME), FILE, + latestGeneration); + currentFile = file; + dirFileFetcher.fetchFile(); + bytesDownloaded += dirFileFetcher.getBytesDownloaded(); + filesDownloaded.add(new HashMap<>(file)); + } + else + { + LOG.info("Skipping download for " + file.get(NAME) + " because it already exists"); + } + } + return bytesDownloaded; + } + + /** + * All the files which are common between master and slave must have same size and same checksum else we assume + * they are not compatible (stale). + * + * @return true if the index stale and we need to download a fresh copy, false otherwise. + * @throws IOException if low level io error + */ + private boolean isIndexStale(Directory dir) throws IOException + { + for (Map file : filesToDownload) + { + String filename = (String) file.get(NAME); + Long length = (Long) file.get(SIZE); + Long checksum = (Long) file.get(CHECKSUM); + if (slowFileExists(dir, filename)) + { + if (checksum != null) + { + if (!(compareFile(dir, filename, length, checksum).equal)) + { + // file exists and size or checksum is different, therefore we must download it again + return true; + } + } + else + { + if (length != dir.fileLength(filename)) + { + LOG.warn("File {} did not match. expected length is {} and actual length is {}", filename, + length, dir.fileLength(filename)); + return true; + } + } + } + } + return false; + } + + /** + * Copy a file by the File#renameTo() method. If it fails, it is considered a failure + *

+ */ + private boolean moveAFile(Directory tmpIdxDir, Directory indexDir, String fname) + { + LOG.debug("Moving file: {}", fname); + boolean success = false; + try + { + if (slowFileExists(indexDir, fname)) + { + LOG.warn("Cannot complete replication attempt because file already exists:" + fname); + + // we fail - we downloaded the files we need, if we can't move one in, we can't + // count on the correct index + return false; + } + } + catch (IOException e) + { + SolrException.log(LOG, "could not check if a file exists", e); + return false; + } + try + { + solrCore.getDirectoryFactory().move(tmpIdxDir, indexDir, fname, DirectoryFactory.IOCONTEXT_NO_CACHE); + success = true; + } + catch (IOException e) + { + SolrException.log(LOG, "Could not move file", e); + } + return success; + } + + /** + * Copy all index files from the temp index dir to the actual index. The segments_N file is copied last. + */ + private boolean moveIndexFiles(Directory tmpIdxDir, Directory indexDir) + { + if (LOG.isDebugEnabled()) + { + try + { + LOG.info("From dir files:" + Arrays.asList(tmpIdxDir.listAll())); + LOG.info("To dir files:" + Arrays.asList(indexDir.listAll())); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + String segmentsFile = null; + for (Map f : filesDownloaded) + { + String fname = (String) f.get(NAME); + // the segments file must be copied last + // or else if there is a failure in between the + // index will be corrupted + if (fname.startsWith("segments_")) + { + //The segments file must be copied in the end + //Otherwise , if the copy fails index ends up corrupted + segmentsFile = fname; + continue; + } + if (!moveAFile(tmpIdxDir, indexDir, fname)) + return false; + } + //copy the segments file last + if (segmentsFile != null) + { + return moveAFile(tmpIdxDir, indexDir, segmentsFile); + } + return true; + } + + /** + *

+ * Copy all the tlog files from the temp tlog dir to the actual tlog dir, and reset + * the {@link UpdateLog}. The copy will try to preserve the original tlog directory + * if the copy fails. + *

+ *

+ * This assumes that the tlog files transferred from the leader are in synch with the + * index files transferred from the leader. The reset of the update log relies on the version + * of the latest operations found in the tlog files. If the tlogs are ahead of the latest commit + * point, it will not copy all the needed buffered updates for the replay and it will miss + * some operations. + *

+ */ + private boolean moveTlogFiles(File tmpTlogDir) + { + UpdateLog ulog = solrCore.getUpdateHandler().getUpdateLog(); + + VersionInfo vinfo = ulog.getVersionInfo(); + vinfo.blockUpdates(); // block updates until the new update log is initialised + try + { + // reset the update log before copying the new tlog directory + CdcrUpdateLog.BufferedUpdates bufferedUpdates = ((CdcrUpdateLog) ulog).resetForRecovery(); + // try to move the temp tlog files to the tlog directory + if (!copyTmpTlogFiles2Tlog(tmpTlogDir)) + return false; + // reinitialise the update log and copy the buffered updates + if (bufferedUpdates.tlog != null) + { + // map file path to its new backup location + File parentDir = FileSystems.getDefault() + .getPath(solrCore.getUpdateHandler().getUpdateLog().getLogDir()).getParent().toFile(); + File backupTlogDir = new File(parentDir, tmpTlogDir.getName()); + bufferedUpdates.tlog = new File(backupTlogDir, bufferedUpdates.tlog.getName()); + } + // init the update log with the new set of tlog files, and copy the buffered updates + ((CdcrUpdateLog) ulog).initForRecovery(bufferedUpdates.tlog, bufferedUpdates.offset); + } + catch (Exception e) + { + LOG.error("Unable to copy tlog files", e); + return false; + } + finally + { + vinfo.unblockUpdates(); + } + return true; + } + + /** + * Make file list + */ + private List makeTmpConfDirFileList(File dir, List fileList) + { + File[] files = dir.listFiles(); + for (File file : files) + { + if (file.isFile()) + { + fileList.add(file); + } + else if (file.isDirectory()) + { + fileList = makeTmpConfDirFileList(file, fileList); + } + } + return fileList; + } + + /** + * The conf files are copied to the tmp dir to the conf dir. A backup of the old file is maintained + */ + private void copyTmpConfFiles2Conf(File tmpconfDir) + { + boolean status; + File confDir = new File(solrCore.getResourceLoader().getConfigDir()); + for (File file : makeTmpConfDirFileList(tmpconfDir, new ArrayList<>())) + { + File oldFile = new File(confDir, + file.getPath().substring(tmpconfDir.getPath().length(), file.getPath().length())); + if (!oldFile.getParentFile().exists()) + { + status = oldFile.getParentFile().mkdirs(); + if (!status) + { + throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to mkdirs: " + oldFile.getParentFile()); + } + } + if (oldFile.exists()) + { + File backupFile = new File(oldFile.getPath() + "." + getDateAsStr(new Date(oldFile.lastModified()))); + if (!backupFile.getParentFile().exists()) + { + status = backupFile.getParentFile().mkdirs(); + if (!status) + { + throw new SolrException(ErrorCode.SERVER_ERROR, + "Unable to mkdirs: " + backupFile.getParentFile()); + } + } + status = oldFile.renameTo(backupFile); + if (!status) + { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unable to rename: " + oldFile + " to: " + backupFile); + } + } + status = file.renameTo(oldFile); + if (!status) + { + throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to rename: " + file + " to: " + oldFile); + } + } + } + + private void copyTmpContentStoreToContentStore(File tmpContentStoreDir, String contentStorePath) throws IOException + { + + String tmpContentStorePath = tmpContentStoreDir.getPath(); + + try + { + Files.walk(tmpContentStoreDir.toPath()).forEach(p -> { + File tmpFile = new File(p.toUri()); + if (!tmpFile.isDirectory()) + { + File csFile = new File(p.toString().replaceFirst(tmpContentStorePath, contentStorePath)); + try + { + Files.createDirectories(Paths.get(csFile.getParent())); + tmpFile.renameTo(csFile); + } + catch (IOException e) + { + LOG.error("impossible to copy {}", csFile.toString()); + } + } + }); + } + catch (IOException e) + { + LOG.error("impossible tmp content store"); + throw e; + } + } + + private void deleteContentStoreFiles(String contentStorePath, List> filesToDelete) + { + filesToDelete.stream().map(f -> (String) f.get(NAME)).forEach(p -> { + File f = new File(contentStorePath, p); + f.delete(); + }); + + LOG.info("deleted {} files from content store", filesToDelete.size()); + } + + private void deleteUnnecessaryContentStoreFiles(String contentStorePath) + { + + AtomicInteger fileDeleted = new AtomicInteger(); + Set fileNames = contentStoreFilesToDownload.stream().map(e -> (String) e.get(NAME)) + .collect(Collectors.toSet()); + try + { + Files.walk(Paths.get(contentStorePath)).forEach(p -> { + File f = new File(p.toUri()); + if (!f.isDirectory() && !fileNames.contains(p.toString().replaceFirst(contentStorePath, ""))) + { + try + { + Files.delete(p); + fileDeleted.getAndIncrement(); + } + catch (IOException ex) + { + LOG.error("Impossible delete file {}", p); + } + } + }); + } + catch (IOException e) + { + LOG.error("Impossible to delete unnecessary files. Content store may contains unused contents"); + } + + LOG.info("deleted {} unnecessary files from content store", fileDeleted); + } + + /** + * The tlog files are moved from the tmp dir to the tlog dir as an atomic filesystem operation. + * A backup of the old directory is maintained. If the directory move fails, it will try to revert back the original + * tlog directory. + */ + private boolean copyTmpTlogFiles2Tlog(File tmpTlogDir) + { + Path tlogDir = FileSystems.getDefault().getPath(solrCore.getUpdateHandler().getUpdateLog().getLogDir()); + Path backupTlogDir = FileSystems.getDefault() + .getPath(tlogDir.getParent().toAbsolutePath().toString(), tmpTlogDir.getName()); + + try + { + Files.move(tlogDir, backupTlogDir, StandardCopyOption.ATOMIC_MOVE); + } + catch (IOException e) + { + SolrException.log(LOG, "Unable to rename: " + tlogDir + " to: " + backupTlogDir, e); + return false; + } + + Path src = FileSystems.getDefault().getPath(backupTlogDir.toAbsolutePath().toString(), tmpTlogDir.getName()); + try + { + Files.move(src, tlogDir, StandardCopyOption.ATOMIC_MOVE); + } + catch (IOException e) + { + SolrException.log(LOG, "Unable to rename: " + src + " to: " + tlogDir, e); + + // In case of error, try to revert back the original tlog directory + try + { + Files.move(backupTlogDir, tlogDir, StandardCopyOption.ATOMIC_MOVE); + } + catch (IOException e2) + { + // bad, we were not able to revert back the original tlog directory + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unable to rename: " + backupTlogDir + " to: " + tlogDir); + } + + return false; + } + + return true; + } + + private String getDateAsStr(Date d) + { + return new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(d); + } + + /** + * The local conf files are compared with the conf files in the master. If they are same (by checksum) do not copy. + * + * @param confFilesToDownload The list of files obtained from master + * @return a list of configuration files which have changed on the master and need to be downloaded. + */ + @SuppressWarnings("unchecked") + private Collection> getModifiedConfFiles(List> confFilesToDownload) + { + if (confFilesToDownload == null || confFilesToDownload.isEmpty()) + return Collections.emptyList(); + //build a map with alias/name as the key + Map> nameVsFile = new HashMap<>(); + NamedList names = new NamedList(); + for (Map map : confFilesToDownload) + { + //if alias is present that is the name the file may have in the slave + String name = (String) (map.get(ALIAS) == null ? map.get(NAME) : map.get(ALIAS)); + nameVsFile.put(name, map); + names.add(name, null); + } + //get the details of the local conf files with the same alias/name + List> localFilesInfo = alfrescoReplicationHandler + .getConfFileInfoFromCache(names, confFileInfoCache); + //compare their size/checksum to see if + for (Map fileInfo : localFilesInfo) + { + String name = (String) fileInfo.get(NAME); + Map m = nameVsFile.get(name); + if (m == null) + continue; // the file is not even present locally (so must be downloaded) + if (m.get(CHECKSUM).equals(fileInfo.get(CHECKSUM))) + { + nameVsFile.remove(name); //checksums are same so the file need not be downloaded + } + } + return nameVsFile.isEmpty() ? Collections.EMPTY_LIST : nameVsFile.values(); + } + + /** + * Stops the ongoing fetch + */ + void abortFetch() + { + stop = true; + } + + @SuppressForbidden(reason = "Need currentTimeMillis for debugging/stats") + private void markReplicationStart() + { + replicationTimer = new RTimer(); + replicationStartTimeStamp = new Date(); + } + + private void markReplicationStop() + { + replicationStartTimeStamp = null; + replicationTimer = null; + } + + Date getReplicationStartTimeStamp() + { + return replicationStartTimeStamp; + } + + long getReplicationTimeElapsed() + { + long timeElapsed = 0; + if (replicationStartTimeStamp != null) + timeElapsed = TimeUnit.SECONDS.convert((long) replicationTimer.getTime(), TimeUnit.MILLISECONDS); + return timeElapsed; + } + + List> getTlogFilesToDownload() + { + //make a copy first because it can be null later + List> tmp = tlogFilesToDownload; + //create a new instance. or else iterator may fail + return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); + } + + List> getTlogFilesDownloaded() + { + //make a copy first because it can be null later + List> tmp = tlogFilesDownloaded; + // NOTE: it's safe to make a copy of a SynchronizedCollection(ArrayList) + return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); + } + + List> getConfFilesToDownload() + { + //make a copy first because it can be null later + List> tmp = confFilesToDownload; + //create a new instance. or else iterator may fail + return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); + } + + List> getConfFilesDownloaded() + { + //make a copy first because it can be null later + List> tmp = confFilesDownloaded; + // NOTE: it's safe to make a copy of a SynchronizedCollection(ArrayList) + return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); + } + + List> getContentStoreFilesDownloaded() + { + //make a copy first because it can be null later + List> tmp = contentStoreFilesDownloaded; + // NOTE: it's safe to make a copy of a SynchronizedCollection(ArrayList) + return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); + } + + List> getFilesToDownload() + { + //make a copy first because it can be null later + List> tmp = filesToDownload; + return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); + } + + List> getContentStoreFileToDownload() + { + //make a copy first because it can be null later + List> tmp = contentStoreFilesToDownload; + return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); + } + + List> getFilesDownloaded() + { + List> tmp = filesDownloaded; + return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); + } + + + Map getCurrentFile() + { + Map tmp = currentFile; + DirectoryFileFetcher tmpFileFetcher = dirFileFetcher; + if (tmp == null) + return null; + tmp = new HashMap<>(tmp); + if (tmpFileFetcher != null) + tmp.put("bytesDownloaded", tmpFileFetcher.getBytesDownloaded()); + return tmp; + } + + NamedList getDetails() throws IOException, SolrServerException + { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(COMMAND, CMD_DETAILS); + params.set("slave", false); + params.set(CommonParams.QT, AlfrescoReplicationHandler.PATH); + + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) + { + client.setSoTimeout(60000); + client.setConnectionTimeout(15000); + QueryRequest request = new QueryRequest(params); + return client.request(request); + } + } + + public void destroy() + { + abortFetch(); + } + + String getMasterUrl() + { + return masterUrl; + } + + private interface FileInterface + { + void sync() throws IOException; + + void write(byte[] buf, int packetSize) throws IOException; + + void close() throws Exception; + + void delete() throws Exception; + } + + public static class IndexFetchResult + { + static final String FAILED_BY_INTERRUPT_MESSAGE = "Fetching index failed by interrupt"; + static final String FAILED_BY_EXCEPTION_MESSAGE = "Fetching index failed by exception"; + /** + * pre-defined results + */ + static final IndexFetchResult ALREADY_IN_SYNC = new IndexFetchResult( + "Local index commit is already in sync with peer", true, null); + static final IndexFetchResult INDEX_FETCH_FAILURE = new IndexFetchResult("Fetching lastest index is failed", + false, null); + static final IndexFetchResult INDEX_FETCH_SUCCESS = new IndexFetchResult("Fetching latest index is successful", + true, null); + static final IndexFetchResult LOCK_OBTAIN_FAILED = new IndexFetchResult("Obtaining SnapPuller lock failed", + false, null); + static final IndexFetchResult MASTER_VERSION_ZERO = new IndexFetchResult( + "Index in peer is empty and never committed yet", true, null); + static final IndexFetchResult NO_INDEX_COMMIT_EXIST = new IndexFetchResult("No IndexCommit in local index", + false, null); + static final IndexFetchResult PEER_INDEX_COMMIT_DELETED = new IndexFetchResult( + "No files to download because IndexCommit in peer was deleted", false, null); + private final String message; + private final boolean successful; + private final Throwable exception; + + IndexFetchResult(String message, boolean successful, Throwable exception) + { + this.message = message; + this.successful = successful; + this.exception = exception; + } + + /* + * @return exception thrown if failed by exception or interrupt, otherwise null + */ + public Throwable getException() + { + return this.exception; + } + + /* + * @return true if index fetch was successful, false otherwise + */ + boolean getSuccessful() + { + return this.successful; + } + + public String getMessage() + { + return this.message; + } + } + + protected static class CompareResult + { + boolean equal = false; + boolean checkSummed = false; + } + + private static class ReplicationHandlerException extends InterruptedException + { + ReplicationHandlerException(String message) + { + super(message); + } + } + + private static class DirectoryFile implements FileInterface + { + private final String saveAs; + private Directory copy2Dir; + private IndexOutput outStream; + + DirectoryFile(Directory tmpIndexDir, String saveAs) throws IOException + { + this.saveAs = saveAs; + this.copy2Dir = tmpIndexDir; + outStream = copy2Dir.createOutput(this.saveAs, DirectoryFactory.IOCONTEXT_NO_CACHE); + } + + public void sync() throws IOException + { + copy2Dir.sync(Collections.singleton(saveAs)); + } + + public void write(byte[] buf, int packetSize) throws IOException + { + outStream.writeBytes(buf, 0, packetSize); + } + + public void close() throws Exception + { + outStream.close(); + } + + public void delete() throws Exception + { + copy2Dir.deleteFile(saveAs); + } + } + + private static class LocalFsFile implements FileInterface + { + FileChannel fileChannel; + File file; + private File copy2Dir; + private FileOutputStream fileOutputStream; + + LocalFsFile(File dir, String saveAs) throws IOException + { + this.copy2Dir = dir; + + this.file = new File(copy2Dir, saveAs); + + File parentDir = this.file.getParentFile(); + if (!parentDir.exists()) + { + if (!parentDir.mkdirs()) + { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to create (sub)directory for file: " + saveAs); + } + } + + this.fileOutputStream = new FileOutputStream(file); + this.fileChannel = this.fileOutputStream.getChannel(); + } + + public void sync() throws IOException + { + FileUtils.sync(file); + } + + public void write(byte[] buf, int packetSize) throws IOException + { + fileChannel.write(ByteBuffer.wrap(buf, 0, packetSize)); + } + + public void close() throws Exception + { + //close the FileOutputStream (which also closes the Channel) + fileOutputStream.close(); + } + + public void delete() throws Exception + { + Files.delete(file.toPath()); + } + } + + /** + * The class acts as a client for ReplicationHandler.FileStream. It understands the protocol of wt=filestream + */ + public class FileFetcher + { + final String solrParamOutput; + final Long indexGen; + final Checksum checksum; + protected FileInterface file; + protected long size; + protected byte[] buf = new byte[1024 * 1024]; + String fileName; + String saveAs; + boolean includeChecksum = true; + long bytesDownloaded = 0; + int errorCount = 0; + boolean aborted = false; + + FileFetcher(FileInterface file, Map fileDetails, String saveAs, String solrParamOutput, long latestGen) + { + this.file = file; + this.fileName = (String) fileDetails.get(NAME); + this.size = (Long) fileDetails.get(SIZE); + this.solrParamOutput = solrParamOutput; + this.saveAs = saveAs; + indexGen = latestGen; + + if (includeChecksum) + { + checksum = new Adler32(); + } + else + { + checksum = null; + } + } + + FileFetcher(String solrParamOutput, long latestGen) + { + + this.solrParamOutput = solrParamOutput; + indexGen = latestGen; + if (includeChecksum) + { + checksum = new Adler32(); + } + else + { + checksum = null; + } + } + + long getBytesDownloaded() + { + return bytesDownloaded; + } + + /** + * The main method which downloads file + */ + void fetchFile() throws Exception + { + bytesDownloaded = 0; + try + { + fetch(); + } + catch (Exception e) + { + if (!aborted) + { + SolrException.log(AlfrescoIndexFetcher.LOG, "Error fetching file, doing one retry...", e); + // one retry + fetch(); + } + else + { + throw e; + } + } + } + + protected void fetch() throws Exception + { + try + { + while (true) + { + final FastInputStream is = getStream(); + int result; + try + { + //fetch packets one by one in a single request + result = fetchPackets(is); + if (result == 0 || result == NO_CONTENT) + { + return; + } + //if there is an error continue. But continue from the point where it got broken + } + finally + { + IOUtils.closeQuietly(is); + } + } + } + finally + { + cleanup(); + //if cleanup succeeds . The file is downloaded fully. do an fsync + fsyncService.submit(() -> { + try + { + file.sync(); + } + catch (IOException e) + { + fsyncException = e; + } + }); + } + } + + protected int fetchPackets(FastInputStream fis) throws Exception + { + byte[] intbytes = new byte[4]; + byte[] longbytes = new byte[8]; + try + { + + while (true) + { + if (stop) + { + stop = false; + aborted = true; + throw new ReplicationHandlerException("User aborted replication"); + } + long checkSumServer = -1; + fis.readFully(intbytes); + //read the size of the packet + int packetSize = readInt(intbytes); + if (packetSize <= 0) + { + LOG.warn("No content received for file: {}", fileName); + return NO_CONTENT; + } + if (buf.length < packetSize) + buf = new byte[packetSize]; + if (checksum != null) + { + //read the checksum + fis.readFully(longbytes); + checkSumServer = readLong(longbytes); + } + //then read the packet of bytes + fis.readFully(buf, 0, packetSize); + //compare the checksum as sent from the master + if (includeChecksum) + { + checksum.reset(); + checksum.update(buf, 0, packetSize); + long checkSumClient = checksum.getValue(); + if (checkSumClient != checkSumServer) + { + LOG.error("Checksum not matched between client and server for file: {}", fileName); + //if checksum is wrong it is a problem return for retry + return 1; + } + } + //if everything is fine, write down the packet to the file + file.write(buf, packetSize); + bytesDownloaded += packetSize; + LOG.debug("Fetched and wrote {} bytes of file: {}", bytesDownloaded, fileName); + if (bytesDownloaded >= size) + return 0; + //errorCount is always set to zero after a successful packet + errorCount = 0; + } + } + catch (ReplicationHandlerException e) + { + throw e; + } + catch (Exception e) + { + LOG.warn("Error in fetching file: {} (downloaded {} of {} bytes)", fileName, bytesDownloaded, size, e); + //for any failure, increment the error count + errorCount++; + //if it fails for the same packet for MAX_RETRIES fail and come out + if (errorCount > MAX_RETRIES) + { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to fetch file: " + fileName + " (downloaded " + bytesDownloaded + " of " + size + + " bytes" + ", error count: " + errorCount + " > " + MAX_RETRIES + ")", e); + } + return ERR; + } + } + + /** + * The webcontainer flushes the data only after it fills the buffer size. So, all data has to be read as readFully() + * other wise it fails. So read everything as bytes and then extract an integer out of it + */ + int readInt(byte[] b) + { + return (((b[0] & 0xff) << 24) | ((b[1] & 0xff) << 16) | ((b[2] & 0xff) << 8) | (b[3] & 0xff)); + + } + + /** + * Same as above but to read longs from a byte array + */ + long readLong(byte[] b) + { + return (((long) (b[0] & 0xff)) << 56) | (((long) (b[1] & 0xff)) << 48) | (((long) (b[2] & 0xff)) << 40) | ( + ((long) (b[3] & 0xff)) << 32) | (((long) (b[4] & 0xff)) << 24) | ((b[5] & 0xff) << 16) | ( + (b[6] & 0xff) << 8) | ((b[7] & 0xff)); + + } + + /** + * cleanup everything + */ + private void cleanup() + { + try + { + file.close(); + } + catch (Exception e) + { + /* no-op */ + LOG.error("Error closing file: {}", this.saveAs, e); + } + if (bytesDownloaded != size) + { + //if the download is not complete then + //delete the file being downloaded + try + { + file.delete(); + } + catch (Exception e) + { + LOG.error("Error deleting file: {}", this.saveAs, e); + } + //if the failure is due to a user abort it is returned normally else an exception is thrown + if (!aborted) + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unable to download " + fileName + " completely. Downloaded " + bytesDownloaded + "!=" + + size); + } + } + + /** + * Open a new stream using HttpClient + */ + protected FastInputStream getStream() throws IOException + { + + ModifiableSolrParams params = new ModifiableSolrParams(); + + // //the method is command=filecontent + params.set(COMMAND, CMD_GET_FILE); + params.set(GENERATION, Long.toString(indexGen)); + params.set(CommonParams.QT, AlfrescoReplicationHandler.PATH); + + //add the version to download. This is used to reserve the download + params.set(solrParamOutput, fileName); + if (useInternalCompression) + { + params.set(COMPRESSION, "true"); + } + + if (this.includeChecksum) + { + params.set(CHECKSUM, true); + } + //wt=filestream this is a custom protocol + params.set(CommonParams.WT, FILE_STREAM); + + // This happen if there is a failure there is a retry. the offset= ensures that + // the server starts from the offset + if (bytesDownloaded > 0) + { + params.set(OFFSET, Long.toString(bytesDownloaded)); + } + + NamedList response; + InputStream is = null; + + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient) + .withResponseParser(null).build()) + { + client.setSoTimeout(60000); + client.setConnectionTimeout(15000); + QueryRequest req = new QueryRequest(params); + response = client.request(req); + is = (InputStream) response.get("stream"); + if (useInternalCompression) + { + is = new InflaterInputStream(is); + } + return new FastInputStream(is); + } + catch (Exception e) + { + //close stream on error + org.apache.commons.io.IOUtils.closeQuietly(is); + throw new IOException("Could not download file '" + fileName + "'", e); + } + } + } + + private class DirectoryFileFetcher extends FileFetcher + { + DirectoryFileFetcher(Directory tmpIndexDir, Map fileDetails, String saveAs, String solrParamOutput, long latestGen) + throws IOException + { + super(new DirectoryFile(tmpIndexDir, saveAs), fileDetails, saveAs, solrParamOutput, latestGen); + } + } + + class LocalFsFileFetcher extends FileFetcher + { + LocalFsFileFetcher(File dir, Map fileDetails, String saveAs, String solrParamOutput, long latestGen) + throws IOException + { + super(new LocalFsFile(dir, saveAs), fileDetails, saveAs, solrParamOutput, latestGen); + } + } + + class ContentStoreFetcher extends FileFetcher + { + private final File dir; + private final Set filesToDownload; + private final Set filesDownloaded; + + ContentStoreFetcher(File dir, String solrParamOutput, List> filesDetails) throws IOException + { + super(solrParamOutput, 0); + this.dir = dir; + this.filesToDownload = new HashSet<>(); + this.filesDownloaded = new HashSet<>(); + filesDetails.forEach(e -> filesToDownload.add((String) e.get(NAME))); + } + + @Override + protected void fetch() throws Exception + { + + while (true) + { + final FastInputStream is = getStream(); + int result; + try + { + //fetch packets one by one in a single request + result = fetchPackets(is); + if (result == 0 || result == NO_CONTENT) + { + return; + } + //if there is an error continue. But continue from the point where it got broken + } + finally + { + IOUtils.closeQuietly(is); + } + } + } + + public void fetchContentStore() throws Exception + { + this.fetchFile(); + } + + @Override + protected FastInputStream getStream() throws IOException + { + + ModifiableSolrParams params = new ModifiableSolrParams(); + + params.set(COMMAND, CMD_CONTENT_STORE_FILES); + params.set(GENERATION, Long.toString(indexGen)); + params.set(CommonParams.QT, AlfrescoReplicationHandler.PATH); + + List l = new ArrayList<>(); + params.set(CONTENT_STORE_FILE_LIST, filesToDownload.toArray(String[]::new)); + + //add the version to download. This is used to reserve the download + // params.set(solrParamOutput, fileName); + if (useInternalCompression) + { + params.set(COMPRESSION, "true"); + } + //use checksum + if (this.includeChecksum) + { + params.set(CHECKSUM, true); + } + //wt=filestream this is a custom protocol + params.set(CommonParams.WT, FILE_STREAM); + // This happen if there is a failure there is a retry. the offset= ensures that + // the server starts from the offset + if (bytesDownloaded > 0) + { + params.set(OFFSET, Long.toString(bytesDownloaded)); + } + + NamedList response; + InputStream is = null; + + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient) + .withResponseParser(null).build()) + { + client.setSoTimeout(60000); + client.setConnectionTimeout(15000); + QueryRequest req = new QueryRequest(params); + response = client.request(req); + is = (InputStream) response.get("stream"); + if (useInternalCompression) + { + is = new InflaterInputStream(is); + } + return new FastInputStream(is); + } + catch (Exception e) + { + //close stream on error + org.apache.commons.io.IOUtils.closeQuietly(is); + throw new IOException("Could not download file '" + fileName + "'", e); + } + } + + @Override + protected int fetchPackets(FastInputStream fis) throws Exception + { + byte[] intbytes = new byte[4]; + byte[] longbytes = new byte[8]; + try + { + while (true) + { + int fileNameSize; + try + { + fis.readFully(intbytes); + fileNameSize = readInt(intbytes); + } + catch (EOFException e) + { + LOG.debug("Fetched the whole batch of files"); + return 0; + } + + byte[] filenameBytes = new byte[fileNameSize]; + fis.readFully(filenameBytes, 0, fileNameSize); + + String fileName = new String(filenameBytes); + + // Error, file not requested + if (!filesToDownload.contains(fileName)) + { + throw new Exception("file " + fileName + " not requested"); + } + + FileInterface file = new LocalFsFile(dir, fileName); + fis.readFully(intbytes); + + int fileSize = readInt(intbytes); + + long fileSizeDownloaded = 0; + if (fileSize == 0) + { + file.close(); + return 0; + } + + while (true) + { + if (stop) + { + stop = false; + aborted = true; + throw new ReplicationHandlerException("User aborted replication"); + } + long checkSumServer = -1; + fis.readFully(intbytes); + //read the size of the packet + int packetSize = readInt(intbytes); + if (packetSize <= 0) + { + LOG.warn("No content received"); + file.close(); + return NO_CONTENT; + } + + if (buf.length < packetSize) + { + buf = new byte[packetSize]; + } + + if (checksum != null) + { + //read the checksum + fis.readFully(longbytes); + checkSumServer = readLong(longbytes); + } + + //then read the packet of bytes + fis.readFully(buf, 0, packetSize); + //compare the checksum as sent from the master + if (includeChecksum) + { + checksum.reset(); + checksum.update(buf, 0, packetSize); + long checkSumClient = checksum.getValue(); + if (checkSumClient != checkSumServer) + { + LOG.warn("Checksum not matched between client and server for file: {}", this.fileName); + //if checksum is wrong it is a problem return for retry + file.close(); + return ERR; + } + } + //if everything is fine, write down the packet to the file + file.write(buf, packetSize); + fileSizeDownloaded += packetSize; + + if (fileSizeDownloaded >= fileSize) + { + file.close(); + bytesDownloaded += fileSizeDownloaded; + + fsyncService.submit(() -> { + try + { + file.sync(); + } + catch (IOException e) + { + fsyncException = e; + } + }); + filesToDownload.remove(fileName); + filesDownloaded.add(fileName); + break; + } + //errorCount is always set to zero after a successful packet + errorCount = 0; + } + + LOG.debug("downloaded content store file: {}", fileName); + } + } + catch (ReplicationHandlerException e) + { + throw e; + } + catch (Exception e) + { + LOG.warn("Error in fetching file: {} (downloaded {} of {} bytes)", + new Object[] { fileName, bytesDownloaded, size, e }); + //for any failure, increment the error count + errorCount++; + //if it fails for the same packet for MAX_RETRIES fail and come out + if (errorCount > MAX_RETRIES) + { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to fetch content store bucket: " + " (downloaded " + filesDownloaded.size() + + " files out of " + filesToDownload.size() + filesDownloaded.size() + + ", error count: " + errorCount + " > " + MAX_RETRIES + ")", e); + } + return ERR; + } + } + + } +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java new file mode 100644 index 000000000..6400050d4 --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java @@ -0,0 +1,2369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + * Modification copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see + * + * + */ +package org.alfresco.solr.handler; + +import org.alfresco.solr.AlfrescoCoreAdminHandler; +import org.alfresco.solr.content.SolrContentStore; +import org.apache.commons.io.IOUtils; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexDeletionPolicy; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.RateLimiter; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.FastOutputStream; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.common.util.SuppressForbidden; +import org.apache.solr.core.CloseHook; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.DirectoryFactory.DirContext; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrDeletionPolicy; +import org.apache.solr.core.SolrEventListener; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.core.backup.repository.LocalFileSystemRepository; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.handler.RestoreCore; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.CdcrUpdateLog; +import org.apache.solr.update.SolrIndexWriter; +import org.apache.solr.update.VersionInfo; +import org.apache.solr.util.DefaultSolrThreadFactory; +import org.apache.solr.util.NumberUtils; +import org.apache.solr.util.PropertiesInputStream; +import org.apache.solr.util.RefCounted; +import org.apache.solr.util.plugin.SolrCoreAware; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.Adler32; +import java.util.zip.Checksum; +import java.util.zip.DeflaterOutputStream; + +import static org.apache.solr.common.params.CommonParams.NAME; + +/** + *

A Handler which provides a REST API for replication and serves replication requests from Slaves.

+ *

When running on the master, it provides the following commands

  1. Get the current replicable index version + * (command=indexversion)
  2. Get the list of files for a given index version + * (command=filelist&indexversion=<VERSION>)
  3. Get full or a part (chunk) of a given index or a config + * file (command=filecontent&file=<FILE_NAME>) You can optionally specify an offset and length to get that + * chunk of the file. You can request a configuration file by using "cf" parameter instead of the "file" parameter.
    4. + *
  4. Get status/statistics (command=details)

When running on the slave, it provides the following + * commands

  1. Perform an index fetch now (command=snappull)
  2. Get status/statistics (command=details)
    3. + *
  3. Abort an index fetch (command=abort)
  4. Enable/Disable polling the master for new versions (command=enablepoll + * or command=disablepoll)
+ * + * @since solr 1.4 + */ +public class AlfrescoReplicationHandler extends RequestHandlerBase implements SolrCoreAware +{ + + public static final String PATH = "/replication"; + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private SolrCore core; + + private SolrContentStore contentStore; + private boolean contentStoreReplication = false; + + public static final class CommitVersionInfo + { + public final long version; + final long generation; + + private CommitVersionInfo(long g, long v) + { + generation = g; + version = v; + } + + @Override + public String toString() + { + return "Commit [version = " + version + ", generation = " + generation + "]"; + } + + /** + * builds a CommitVersionInfo data for the specified IndexCommit. + * Will never be null, ut version and generation may be zero if + * there are problems extracting them from the commit data + */ + public static CommitVersionInfo build(IndexCommit commit) + { + long generation = commit.getGeneration(); + long version = 0; + try + { + final Map commitData = commit.getUserData(); + String commitTime = commitData.get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY); + if (commitTime != null) + { + try + { + version = Long.parseLong(commitTime); + } + catch (NumberFormatException e) + { + LOG.warn("Version in commitData was not formatted correctly: " + commitTime, e); + } + } + } + catch (IOException e) + { + LOG.warn("Unable to get version from commitData, commit: " + commit, e); + } + return new CommitVersionInfo(generation, version); + } + } + + private AlfrescoIndexFetcher pollingAlfrescoIndexFetcher; + + private ReentrantLock indexFetchLock = new ReentrantLock(); + + private static Lock contentStoreReplicationLock = new ReentrantLock(); + + private static boolean isContentStoreReplicating = false; + + private ExecutorService restoreExecutor = ExecutorUtil + .newMDCAwareSingleThreadExecutor(new DefaultSolrThreadFactory("restoreExecutor")); + + private volatile Future restoreFuture; + + private volatile String currentRestoreName; + + private String includeConfFiles; + + private NamedList confFileNameAlias = new NamedList<>(); + + private boolean isMaster; + + private boolean isSlave; + + private boolean replicateOnOptimize; + + private boolean replicateOnCommit; + + private boolean replicateOnStart; + + private ScheduledExecutorService executorService; + + private volatile long executorStartTime; + + private int numberBackupsToKeep = 0; //zero: do not delete old backups + + private int numTimesReplicated = 0; + + private final Map confFileInfoCache = new HashMap<>(); + + private Integer reserveCommitDuration = readIntervalMs("00:00:10"); + + volatile IndexCommit indexCommitPoint; + + volatile NamedList snapShootDetails; + + private AtomicBoolean replicationEnabled = new AtomicBoolean(true); + + private Long pollIntervalNs; + private String pollIntervalStr; + + /** + * Disable the timer task for polling + */ + private AtomicBoolean pollDisabled = new AtomicBoolean(false); + + private String getPollInterval() + { + return pollIntervalStr; + } + + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception + { + rsp.setHttpCaching(false); + final SolrParams solrParams = req.getParams(); + String command = solrParams.get(COMMAND); + if (command == null) + { + rsp.add(STATUS, OK_STATUS); + rsp.add("message", "No command"); + return; + } + // This command does not give the current index version of the master + // It gives the current 'replicateable' index version + if (command.equals(CMD_INDEX_VERSION)) + { + IndexCommit commitPoint = indexCommitPoint; // make a copy so it won't change + + if (commitPoint == null) + { + // if this handler is 'lazy', we may not have tracked the last commit + // because our commit listener is registered on inform + commitPoint = core.getDeletionPolicy().getLatestCommit(); + } + + if (commitPoint != null && replicationEnabled.get()) + { + // + // There is a race condition here. The commit point may be changed / deleted by the time + // we get around to reserving it. This is a very small window though, and should not result + // in a catastrophic failure, but will result in the client getting an empty file list for + // the CMD_GET_FILE_LIST command. + // + core.getDeletionPolicy().setReserveDuration(commitPoint.getGeneration(), reserveCommitDuration); + rsp.add(CMD_INDEX_VERSION, IndexDeletionPolicyWrapper.getCommitTimestamp(commitPoint)); + rsp.add(GENERATION, commitPoint.getGeneration()); + rsp.add(CONTENT_STORE_VERSION, contentStore.getLastCommittedVersion()); + } + else + { + // This happens when replication is not configured to happen after startup and no commit/optimize + // has happened yet. + rsp.add(CMD_INDEX_VERSION, 0L); + rsp.add(GENERATION, 0L); + } + } + else if (command.equals(CMD_GET_FILE)) + { + getFileStream(solrParams, rsp); + } + else if (command.equals(CMD_CONTENT_STORE_FILES)) + { + getContetntStore(new ModifiableSolrParams(solrParams), rsp); + } + else if (command.equals(CMD_GET_FILE_LIST)) + { + getFileList(solrParams, rsp); + } + else if (command.equalsIgnoreCase(CMD_BACKUP)) + { + doSnapShoot(new ModifiableSolrParams(solrParams), rsp, req); + rsp.add(STATUS, OK_STATUS); + } + else if (command.equalsIgnoreCase(CMD_RESTORE)) + { + restore(new ModifiableSolrParams(solrParams), rsp, req); + rsp.add(STATUS, OK_STATUS); + } + else if (command.equalsIgnoreCase(CMD_RESTORE_STATUS)) + { + rsp.add(CMD_RESTORE_STATUS, getRestoreStatus()); + } + else if (command.equalsIgnoreCase(CMD_DELETE_BACKUP)) + { + deleteSnapshot(new ModifiableSolrParams(solrParams)); + rsp.add(STATUS, OK_STATUS); + } + else if (command.equalsIgnoreCase(CMD_FETCH_INDEX)) + { + String masterUrl = solrParams.get(MASTER_URL); + if (!isSlave && masterUrl == null) + { + rsp.add(STATUS, ERR_STATUS); + rsp.add("message", "No slave configured or no 'masterUrl' Specified"); + return; + } + final SolrParams paramsCopy = new ModifiableSolrParams(solrParams); + Thread fetchThread = new Thread(() -> doFetch(paramsCopy, false), "explicit-fetchindex-cmd"); + fetchThread.setDaemon(false); + fetchThread.start(); + if (solrParams.getBool(WAIT, false)) + { + fetchThread.join(); + } + rsp.add(STATUS, OK_STATUS); + } + else if (command.equalsIgnoreCase(CMD_DISABLE_POLL)) + { + if (pollingAlfrescoIndexFetcher != null) + { + disablePoll(); + rsp.add(STATUS, OK_STATUS); + } + else + { + rsp.add(STATUS, ERR_STATUS); + rsp.add("message", "No slave configured"); + } + } + else if (command.equalsIgnoreCase(CMD_ENABLE_POLL)) + { + if (pollingAlfrescoIndexFetcher != null) + { + enablePoll(); + rsp.add(STATUS, OK_STATUS); + } + else + { + rsp.add(STATUS, ERR_STATUS); + rsp.add("message", "No slave configured"); + } + } + else if (command.equalsIgnoreCase(CMD_ABORT_FETCH)) + { + if (abortFetch()) + { + rsp.add(STATUS, OK_STATUS); + } + else + { + rsp.add(STATUS, ERR_STATUS); + rsp.add("message", "No slave configured"); + } + } + else if (command.equals(CMD_SHOW_COMMITS)) + { + rsp.add(CMD_SHOW_COMMITS, getCommits()); + } + else if (command.equals(CMD_DETAILS)) + { + rsp.add(CMD_DETAILS, getReplicationDetails(solrParams.getBool("slave", true))); + } + else if (CMD_ENABLE_REPL.equalsIgnoreCase(command)) + { + replicationEnabled.set(true); + rsp.add(STATUS, OK_STATUS); + } + else if (CMD_DISABLE_REPL.equalsIgnoreCase(command)) + { + replicationEnabled.set(false); + rsp.add(STATUS, OK_STATUS); + } + } + + private boolean abortFetch() + { + AlfrescoIndexFetcher fetcher = currentAlfrescoIndexFetcher; + if (fetcher != null) + { + fetcher.abortFetch(); + return true; + } + else + { + return false; + } + } + + private void deleteSnapshot(ModifiableSolrParams params) + { + String name = params.get(NAME); + if (name == null) + { + throw new SolrException(ErrorCode.BAD_REQUEST, "Missing mandatory param: name"); + } + + SnapShooter snapShooter = new SnapShooter(core, params.get(CoreAdminParams.BACKUP_LOCATION), params.get(NAME)); + snapShooter.validateDeleteSnapshot(); + snapShooter.deleteSnapAsync(this); + } + + private List> getCommits() + { + Map commits = core.getDeletionPolicy().getCommits(); + List> l = new ArrayList<>(); + + for (IndexCommit c : commits.values()) + { + try + { + NamedList nl = new NamedList<>(); + nl.add("indexVersion", IndexDeletionPolicyWrapper.getCommitTimestamp(c)); + nl.add(GENERATION, c.getGeneration()); + List commitList = new ArrayList<>(c.getFileNames().size()); + commitList.addAll(c.getFileNames()); + Collections.sort(commitList); + nl.add(CMD_GET_FILE_LIST, commitList); + l.add(nl); + } + catch (IOException e) + { + LOG.warn("Exception while reading files for commit " + c, e); + } + } + return l; + } + + static long getCheckSum(Checksum checksum, File f) + { + checksum.reset(); + byte[] buffer = new byte[1024 * 1024]; + int bytesRead; + try (FileInputStream fis = new FileInputStream(f)) + { + while ((bytesRead = fis.read(buffer)) >= 0) + { + checksum.update(buffer, 0, bytesRead); + } + return checksum.getValue(); + } + catch (Exception e) + { + LOG.warn("Exception in finding checksum of " + f, e); + throw new RuntimeException(e); + } + } + + private volatile AlfrescoIndexFetcher currentAlfrescoIndexFetcher; + + boolean acquireContentStoreReplicationTask() + { + contentStoreReplicationLock.lock(); + if (!isContentStoreReplicating) + { + contentStoreReplication = true; + isContentStoreReplicating = true; + } + + contentStoreReplicationLock.unlock(); + return contentStoreReplication; + } + + private void releaseContentStoreReplicationTask() + { + if (contentStoreReplication) + { + contentStoreReplicationLock.lock(); + isContentStoreReplicating = false; + contentStoreReplication = false; + contentStoreReplicationLock.unlock(); + } + } + + private AlfrescoIndexFetcher.IndexFetchResult doFetch(SolrParams solrParams, boolean forceReplication) + { + String masterUrl = solrParams == null ? null : solrParams.get(MASTER_URL); + if (!indexFetchLock.tryLock()) + return AlfrescoIndexFetcher.IndexFetchResult.LOCK_OBTAIN_FAILED; + try + { + if (masterUrl != null) + { + if (currentAlfrescoIndexFetcher != null && currentAlfrescoIndexFetcher != pollingAlfrescoIndexFetcher) + { + currentAlfrescoIndexFetcher.destroy(); + } + currentAlfrescoIndexFetcher = new AlfrescoIndexFetcher(solrParams.toNamedList(), this, core, contentStore); + } + else + { + currentAlfrescoIndexFetcher = pollingAlfrescoIndexFetcher; + } + + return currentAlfrescoIndexFetcher.fetchLatestIndex(forceReplication, acquireContentStoreReplicationTask()); + } + catch (Exception e) + { + SolrException.log(LOG, "Index fetch failed ", e); + return new AlfrescoIndexFetcher.IndexFetchResult(AlfrescoIndexFetcher.IndexFetchResult.FAILED_BY_EXCEPTION_MESSAGE, false, + e); + } + finally + { + if (pollingAlfrescoIndexFetcher != null) + { + currentAlfrescoIndexFetcher = pollingAlfrescoIndexFetcher; + } + + releaseContentStoreReplicationTask(); + indexFetchLock.unlock(); + } + } + + private boolean isReplicating() + { + return indexFetchLock.isLocked(); + } + + private void restore(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) throws IOException + { + if (restoreFuture != null && !restoreFuture.isDone()) + { + throw new SolrException(ErrorCode.BAD_REQUEST, + "Restore in progress. Cannot run multiple restore operations" + "for the same core"); + } + + String name = params.get(NAME); + String location = params.get(CoreAdminParams.BACKUP_LOCATION); + + String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY); + CoreContainer cc = core.getCoreContainer(); + BackupRepository repo; + if (repoName != null) + { + repo = cc.newBackupRepository(Optional.of(repoName)); + location = repo.getBackupLocation(location); + if (location == null) + { + throw new IllegalArgumentException("location is required"); + } + } + else + { + repo = new LocalFileSystemRepository(); + } + + //If location is not provided then assume that the restore index is present inside the data directory. + if (location == null) + { + location = core.getDataDir(); + } + + URI locationUri = repo.createURI(location); + + //If name is not provided then look for the last unnamed( the ones with the snapshot.timestamp format) + //snapshot folder since we allow snapshots to be taken without providing a name. Pick the latest timestamp. + if (name == null) + { + String[] filePaths = repo.listAll(locationUri); + List dirs = new ArrayList<>(); + for (String f : filePaths) + { + OldBackupDirectory obd = new OldBackupDirectory(locationUri, f); + if (obd.getTimestamp().isPresent()) + { + dirs.add(obd); + } + } + Collections.sort(dirs); + if (dirs.size() == 0) + { + throw new SolrException(ErrorCode.BAD_REQUEST, + "No backup name specified and none found in " + core.getDataDir()); + } + name = dirs.get(0).getDirName(); + } + else + { + //"snapshot." is prefixed by snapshooter + name = "snapshot." + name; + } + + RestoreCore restoreCore = new RestoreCore(repo, core, locationUri, name); + try + { + MDC.put("RestoreCore.core", core.getName()); + MDC.put("RestoreCore.backupLocation", location); + MDC.put("RestoreCore.backupName", name); + restoreFuture = restoreExecutor.submit(restoreCore); + currentRestoreName = name; + } + finally + { + MDC.remove("RestoreCore.core"); + MDC.remove("RestoreCore.backupLocation"); + MDC.remove("RestoreCore.backupName"); + } + } + + private NamedList getRestoreStatus() + { + NamedList status = new SimpleOrderedMap<>(); + + if (restoreFuture == null) + { + status.add(STATUS, "No restore actions in progress"); + return status; + } + + status.add("snapshotName", currentRestoreName); + if (restoreFuture.isDone()) + { + try + { + boolean success = restoreFuture.get(); + if (success) + { + status.add(STATUS, SUCCESS); + } + else + { + status.add(STATUS, FAILED); + } + } + catch (Exception e) + { + status.add(STATUS, FAILED); + status.add(EXCEPTION, e.getMessage()); + } + } + else + { + status.add(STATUS, "In Progress"); + } + return status; + } + + private void doSnapShoot(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) + { + try + { + int numberToKeep = params.getInt(NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM, 0); + if (numberToKeep > 0 && numberBackupsToKeep > 0) + { + throw new SolrException(ErrorCode.BAD_REQUEST, + "Cannot use " + NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM + " if " + + NUMBER_BACKUPS_TO_KEEP_INIT_PARAM + " was specified in the configuration."); + } + numberToKeep = Math.max(numberToKeep, numberBackupsToKeep); + if (numberToKeep < 1) + { + numberToKeep = Integer.MAX_VALUE; + } + + IndexCommit indexCommit; + String commitName = params.get(CoreAdminParams.COMMIT_NAME); + if (commitName != null) + { + SolrSnapshotMetaDataManager snapshotMgr = core.getSnapshotMetaDataManager(); + Optional commit = snapshotMgr.getIndexCommitByName(commitName); + if (commit.isPresent()) + { + indexCommit = commit.get(); + } + else + { + throw new SolrException(ErrorCode.BAD_REQUEST, + "Unable to find an index commit with name " + commitName + " for core " + core.getName()); + } + } + else + { + IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy(); + indexCommit = delPolicy.getLatestCommit(); + + if (indexCommit == null) + { + indexCommit = req.getSearcher().getIndexReader().getIndexCommit(); + } + } + + String location = params.get(CoreAdminParams.BACKUP_LOCATION); + String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY); + CoreContainer cc = core.getCoreContainer(); + BackupRepository repo; + if (repoName != null) + { + repo = cc.newBackupRepository(Optional.of(repoName)); + location = repo.getBackupLocation(location); + if (location == null) + { + throw new IllegalArgumentException("location is required"); + } + } + else + { + repo = new LocalFileSystemRepository(); + if (location == null) + { + location = core.getDataDir(); + } + else + { + location = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString(); + } + } + + // small race here before the commit point is saved + URI locationUri = repo.createURI(location); + SnapShooter snapShooter = new SnapShooter(repo, core, locationUri, params.get(NAME), commitName); + snapShooter.validateCreateSnapshot(); + snapShooter.createSnapAsync(indexCommit, numberToKeep, (nl) -> snapShootDetails = nl); + + } + catch (Exception e) + { + LOG.warn("Exception during creating a snapshot", e); + rsp.add("exception", e); + } + } + + /** + * This method adds an Object of FileStream to the response . The FileStream implements a custom protocol which is + * understood by IndexFetcher.FileFetcher + */ + private void getFileStream(SolrParams solrParams, SolrQueryResponse rsp) + { + ModifiableSolrParams rawParams = new ModifiableSolrParams(solrParams); + rawParams.set(CommonParams.WT, FILE_STREAM); + + String cfileName = solrParams.get(CONF_FILE_SHORT); + String tlogFileName = solrParams.get(TLOG_FILE); + if (cfileName != null) + { + rsp.add(FILE_STREAM, new LocalFsConfFileStream(solrParams)); + } + else if (tlogFileName != null) + { + rsp.add(FILE_STREAM, new LocalFsTlogFileStream(solrParams)); + } + else + { + rsp.add(FILE_STREAM, new DirectoryFileStream(solrParams)); + } + } + + private void getContetntStore(SolrParams solrParams, SolrQueryResponse rsp) + { + rsp.add(FILE_STREAM, new ContentStoreFilesStream(solrParams)); + } + + private void getFileList(SolrParams solrParams, SolrQueryResponse rsp) + { + String v = solrParams.get(GENERATION); + + if (v == null) + { + rsp.add("status", "no index generation specified"); + return; + } + + long indexGeneration = Long.parseLong(v); + + v = solrParams.get(CONTENT_STORE_VERSION); + if (v == null) + { + rsp.add("status", "no content store generation specified"); + return; + } + + long contentStoreGeneration = Long.parseLong(v); + + if (indexGeneration != NO_INDEX_REPLICATION_REQUIRED) + { + + IndexCommit commit = core.getDeletionPolicy().getCommitPoint(indexGeneration); + + if (commit == null) + { + rsp.add("status", "invalid index generation"); + return; + } + + // reserve the indexcommit for sometime + core.getDeletionPolicy().setReserveDuration(indexGeneration, reserveCommitDuration); + List> result = new ArrayList<>(); + Directory dir = null; + try + { + dir = core.getDirectoryFactory() + .get(core.getNewIndexDir(), DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); + SegmentInfos infos = SegmentInfos.readCommit(dir, commit.getSegmentsFileName()); + + for (SegmentCommitInfo commitInfo : infos) + { + for (String file : commitInfo.files()) + { + Map fileMeta = new HashMap<>(); + fileMeta.put(NAME, file); + fileMeta.put(SIZE, dir.fileLength(file)); + + try (final IndexInput in = dir.openInput(file, IOContext.READONCE)) + { + try + { + long checksum = CodecUtil.retrieveChecksum(in); + fileMeta.put(CHECKSUM, checksum); + } + catch (Exception e) + { + LOG.warn("Could not read checksum from index file: " + file, e); + } + } + + result.add(fileMeta); + } + } + + // add the segments_N file + + Map fileMeta = new HashMap<>(); + fileMeta.put(NAME, infos.getSegmentsFileName()); + fileMeta.put(SIZE, dir.fileLength(infos.getSegmentsFileName())); + if (infos.getId() != null) + { + try ( + final IndexInput in = dir.openInput(infos.getSegmentsFileName(), IOContext.READONCE)) + { + try + { + fileMeta.put(CHECKSUM, CodecUtil.retrieveChecksum(in)); + } + catch (Exception e) + { + LOG.warn("Could not read checksum from index file: " + infos.getSegmentsFileName(), e); + } + } + } + result.add(fileMeta); + } + catch (IOException e) + { + rsp.add("status", "unable to get file names for given index generation"); + rsp.add(EXCEPTION, e); + LOG.error("Unable to get file names for indexCommit generation: " + indexGeneration, e); + } + finally + { + if (dir != null) + { + try + { + core.getDirectoryFactory().release(dir); + } + catch (IOException e) + { + SolrException.log(LOG, "Could not release directory after fetching file list", e); + } + } + } + rsp.add(CMD_GET_FILE_LIST, result); + + // fetch list of tlog files only if cdcr is activated + if (solrParams.getBool(TLOG_FILES, true) && core.getUpdateHandler().getUpdateLog() != null && core + .getUpdateHandler().getUpdateLog() instanceof CdcrUpdateLog) + { + try + { + List> tlogfiles = getTlogFileList(commit); + LOG.info("Adding tlog files to list: " + tlogfiles); + rsp.add(TLOG_FILES, tlogfiles); + } + catch (IOException e) + { + rsp.add("status", "unable to get tlog file names for given index generation"); + rsp.add(EXCEPTION, e); + LOG.error("Unable to get tlog file names for indexCommit generation: " + indexGeneration, e); + } + } + + if (confFileNameAlias.size() < 1 || core.getCoreContainer().isZooKeeperAware()) + { + return; + } + LOG.debug("Adding config files to list: " + includeConfFiles); + //if configuration files need to be included get their details + rsp.add(CONF_FILES, getConfFileInfoFromCache(confFileNameAlias, confFileInfoCache)); + } + + if (contentStoreGeneration != SolrContentStore.NO_CONTENT_STORE_REPLICATION_REQUIRED) + { + Map>> changes = contentStore.getChanges(contentStoreGeneration); + rsp.add(CONTENT_STORE_FILES, changes); + } + } + + /** + * Retrieves the list of tlog files associated to a commit point. + */ + private List> getTlogFileList(IndexCommit commit) throws IOException + { + long maxVersion = this.getMaxVersion(commit); + CdcrUpdateLog ulog = (CdcrUpdateLog) core.getUpdateHandler().getUpdateLog(); + String[] logList = ulog.getLogList(new File(ulog.getLogDir())); + List> tlogFiles = new ArrayList<>(); + for (String fileName : logList) + { + // filter out tlogs that are older than the current index commit generation, so that the list of tlog files is + // in synch with the latest index commit point + long startVersion = Math.abs(Long.parseLong(fileName.substring(fileName.lastIndexOf('.') + 1))); + if (startVersion < maxVersion) + { + Map fileMeta = new HashMap<>(); + fileMeta.put(NAME, fileName); + fileMeta.put(SIZE, new File(ulog.getLogDir(), fileName).length()); + tlogFiles.add(fileMeta); + } + } + return tlogFiles; + } + + /** + * Retrieves the maximum version number from an index commit. + */ + private long getMaxVersion(IndexCommit commit) throws IOException + { + try (DirectoryReader reader = DirectoryReader.open(commit)) + { + IndexSearcher searcher = new IndexSearcher(reader); + VersionInfo vinfo = core.getUpdateHandler().getUpdateLog().getVersionInfo(); + return Math.abs(vinfo.getMaxVersionFromIndex(searcher)); + } + } + + /** + * For configuration files, checksum of the file is included because, unlike index files, they may have same content + * but different timestamps. + *

+ * The local conf files information is cached so that everytime it does not have to compute the checksum. The cache is + * refreshed only if the lastModified of the file changes + */ + List> getConfFileInfoFromCache(NamedList nameAndAlias, final Map confFileInfoCache) + { + List> confFiles = new ArrayList<>(); + synchronized (confFileInfoCache) + { + File confDir = new File(core.getResourceLoader().getConfigDir()); + Checksum checksum = null; + for (int i = 0; i < nameAndAlias.size(); i++) + { + String cf = nameAndAlias.getName(i); + File f = new File(confDir, cf); + if (!f.exists() || f.isDirectory()) + continue; //must not happen + FileInfo info = confFileInfoCache.get(cf); + if (info == null || info.lastmodified != f.lastModified() || info.size != f.length()) + { + if (checksum == null) + checksum = new Adler32(); + info = new FileInfo(f.lastModified(), cf, f.length(), getCheckSum(checksum, f)); + confFileInfoCache.put(cf, info); + } + Map m = info.getAsMap(); + if (nameAndAlias.getVal(i) != null) + m.put(ALIAS, nameAndAlias.getVal(i)); + confFiles.add(m); + } + } + return confFiles; + } + + public static class FileInfo + { + long lastmodified; + String name; + long size; + long checksum; + + public FileInfo(File file, String name) + { + Checksum checksum = new Adler32(); + + this.lastmodified = file.lastModified(); + this.name = name; + this.size = file.length(); + this.checksum = getCheckSum(checksum, file); + } + + FileInfo(long lasmodified, String name, long size, long checksum) + { + this.lastmodified = lasmodified; + this.name = name; + this.size = size; + this.checksum = checksum; + } + + public Map getAsMap() + { + Map map = new HashMap<>(); + map.put(NAME, name); + map.put(SIZE, size); + map.put(CHECKSUM, checksum); + return map; + } + } + + private void disablePoll() + { + if (isSlave) + { + pollDisabled.set(true); + LOG.info("inside disable poll, value of pollDisabled = " + pollDisabled); + } + } + + private void enablePoll() + { + if (isSlave) + { + pollDisabled.set(false); + LOG.info("inside enable poll, value of pollDisabled = " + pollDisabled); + } + } + + private boolean isPollingDisabled() + { + return pollDisabled.get(); + } + + @SuppressForbidden(reason = "Need currentTimeMillis, to output next execution time in replication details") + private void markScheduledExecutionStart() + { + executorStartTime = System.currentTimeMillis(); + } + + private Date getNextScheduledExecTime() + { + Date nextTime = null; + if (executorStartTime > 0) + nextTime = new Date( + executorStartTime + TimeUnit.MILLISECONDS.convert(pollIntervalNs, TimeUnit.NANOSECONDS)); + return nextTime; + } + + @SuppressWarnings("unused") + int getTimesReplicatedSinceStartup() + { + return numTimesReplicated; + } + + @SuppressWarnings("unused") + void setTimesReplicatedSinceStartup() + { + numTimesReplicated++; + } + + @Override + public Category getCategory() + { + return Category.REPLICATION; + } + + @Override + public String getDescription() + { + return "ReplicationHandler provides replication of index and configuration files from Master to Slaves"; + } + + /** + * returns the CommitVersionInfo for the current searcher, or null on error. + */ + private CommitVersionInfo getIndexVersion() + { + CommitVersionInfo v = null; + RefCounted searcher = core.getSearcher(); + try + { + v = CommitVersionInfo.build(searcher.get().getIndexReader().getIndexCommit()); + } + catch (IOException e) + { + LOG.warn("Unable to get index commit: ", e); + } + finally + { + searcher.decref(); + } + return v; + } + + @Override + @SuppressWarnings("unchecked") + public NamedList getStatistics() + { + NamedList list = super.getStatistics(); + if (core != null) + { + list.add("indexSize", NumberUtils.readableSize(core.getIndexSize())); + CommitVersionInfo vInfo = (core != null && !core.isClosed()) ? getIndexVersion() : null; + list.add("indexVersion", null == vInfo ? 0 : vInfo.version); + list.add(GENERATION, null == vInfo ? 0 : vInfo.generation); + + list.add("indexPath", core.getIndexDir()); + list.add("isMaster", String.valueOf(isMaster)); + list.add("isSlave", String.valueOf(isSlave)); + + AlfrescoIndexFetcher fetcher = currentAlfrescoIndexFetcher; + if (fetcher != null) + { + list.add(MASTER_URL, fetcher.getMasterUrl()); + if (getPollInterval() != null) + { + list.add(POLL_INTERVAL, getPollInterval()); + } + list.add("isPollingDisabled", String.valueOf(isPollingDisabled())); + list.add("isReplicating", String.valueOf(isReplicating())); + long elapsed = fetcher.getReplicationTimeElapsed(); + long val = fetcher.getTotalBytesDownloaded(); + if (elapsed > 0) + { + list.add("timeElapsed", elapsed); + list.add("bytesDownloaded", val); + list.add("downloadSpeed", val / elapsed); + } + Properties props = loadReplicationProperties(); + addVal(list, AlfrescoIndexFetcher.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class); + addVal(list, AlfrescoIndexFetcher.INDEX_REPLICATED_AT, props, Date.class); + addVal(list, AlfrescoIndexFetcher.CONF_FILES_REPLICATED_AT, props, Date.class); + addVal(list, AlfrescoIndexFetcher.REPLICATION_FAILED_AT, props, Date.class); + addVal(list, AlfrescoIndexFetcher.TIMES_FAILED, props, Integer.class); + addVal(list, AlfrescoIndexFetcher.TIMES_INDEX_REPLICATED, props, Integer.class); + addVal(list, AlfrescoIndexFetcher.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class); + addVal(list, AlfrescoIndexFetcher.TIMES_CONFIG_REPLICATED, props, Integer.class); + addVal(list, AlfrescoIndexFetcher.CONF_FILES_REPLICATED, props, String.class); + } + if (isMaster) + { + if (includeConfFiles != null) + list.add("confFilesToReplicate", includeConfFiles); + list.add(REPLICATE_AFTER, getReplicateAfterStrings()); + list.add("replicationEnabled", String.valueOf(replicationEnabled.get())); + } + } + return list; + } + + /** + * Used for showing statistics and progress information. + */ + private NamedList getReplicationDetails(boolean showSlaveDetails) + { + NamedList details = new SimpleOrderedMap<>(); + NamedList master = new SimpleOrderedMap<>(); + NamedList slave = new SimpleOrderedMap<>(); + + details.add("indexSize", NumberUtils.readableSize(core.getIndexSize())); + details.add("indexPath", core.getIndexDir()); + details.add(CMD_SHOW_COMMITS, getCommits()); + details.add("isMaster", String.valueOf(isMaster)); + details.add("isSlave", String.valueOf(isSlave)); + CommitVersionInfo vInfo = getIndexVersion(); + details.add("indexVersion", null == vInfo ? 0 : vInfo.version); + details.add("robavaria", 50); + details.add(GENERATION, null == vInfo ? 0 : vInfo.generation); + + IndexCommit commit = indexCommitPoint; // make a copy so it won't change + + if (isMaster) + { + if (includeConfFiles != null) + master.add(CONF_FILES, includeConfFiles); + master.add(REPLICATE_AFTER, getReplicateAfterStrings()); + master.add("replicationEnabled", String.valueOf(replicationEnabled.get())); + } + + if (isMaster && commit != null) + { + CommitVersionInfo repCommitInfo = CommitVersionInfo.build(commit); + master.add("replicableVersion", repCommitInfo.version); + master.add("replicableGeneration", repCommitInfo.generation); + } + + AlfrescoIndexFetcher fetcher = currentAlfrescoIndexFetcher; + if (fetcher != null) + { + Properties props = loadReplicationProperties(); + if (showSlaveDetails) + { + try + { + NamedList nl = fetcher.getDetails(); + slave.add("masterDetails", nl.get(CMD_DETAILS)); + } + catch (Exception e) + { + LOG.warn("Exception while invoking 'details' method for replication on master ", e); + slave.add(ERR_STATUS, "invalid_master"); + } + } + slave.add(MASTER_URL, fetcher.getMasterUrl()); + if (getPollInterval() != null) + { + slave.add(POLL_INTERVAL, getPollInterval()); + } + Date nextScheduled = getNextScheduledExecTime(); + if (nextScheduled != null && !isPollingDisabled()) + { + slave.add(NEXT_EXECUTION_AT, nextScheduled.toString()); + } + else if (isPollingDisabled()) + { + slave.add(NEXT_EXECUTION_AT, "Polling disabled"); + } + addVal(slave, AlfrescoIndexFetcher.INDEX_REPLICATED_AT, props, Date.class); + addVal(slave, AlfrescoIndexFetcher.INDEX_REPLICATED_AT_LIST, props, List.class); + addVal(slave, AlfrescoIndexFetcher.REPLICATION_FAILED_AT_LIST, props, List.class); + addVal(slave, AlfrescoIndexFetcher.TIMES_INDEX_REPLICATED, props, Integer.class); + addVal(slave, AlfrescoIndexFetcher.CONF_FILES_REPLICATED, props, Integer.class); + addVal(slave, AlfrescoIndexFetcher.TIMES_CONFIG_REPLICATED, props, Integer.class); + addVal(slave, AlfrescoIndexFetcher.CONF_FILES_REPLICATED_AT, props, Integer.class); + addVal(slave, AlfrescoIndexFetcher.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class); + addVal(slave, AlfrescoIndexFetcher.TIMES_FAILED, props, Integer.class); + addVal(slave, AlfrescoIndexFetcher.REPLICATION_FAILED_AT, props, Date.class); + addVal(slave, AlfrescoIndexFetcher.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class); + + slave.add("currentDate", new Date().toString()); + slave.add("isPollingDisabled", String.valueOf(isPollingDisabled())); + boolean isReplicating = isReplicating(); + slave.add("isReplicating", String.valueOf(isReplicating)); + if (isReplicating) + { + try + { + long bytesToDownload = 0; + List filesToDownload = new ArrayList<>(); + for (Map file : fetcher.getFilesToDownload()) + { + filesToDownload.add((String) file.get(NAME)); + bytesToDownload += (Long) file.get(SIZE); + } + + //get list of conf files to download + for (Map file : fetcher.getConfFilesToDownload()) + { + filesToDownload.add((String) file.get(NAME)); + bytesToDownload += (Long) file.get(SIZE); + } + + //get list of conf files to download + for (Map file : fetcher.getContentStoreFileToDownload()) + { + filesToDownload.add((String) file.get(NAME)); + bytesToDownload += (Long) file.get(SIZE); + } + + slave.add("filesToDownload", filesToDownload); + slave.add("numFilesToDownload", String.valueOf(filesToDownload.size())); + slave.add("bytesToDownload", NumberUtils.readableSize(bytesToDownload)); + + long bytesDownloaded = 0; + List filesDownloaded = new ArrayList<>(); + for (Map file : fetcher.getFilesDownloaded()) + { + filesDownloaded.add((String) file.get(NAME)); + bytesDownloaded += (Long) file.get(SIZE); + } + + //get list of conf files downloaded + for (Map file : fetcher.getConfFilesDownloaded()) + { + filesDownloaded.add((String) file.get(NAME)); + bytesDownloaded += (Long) file.get(SIZE); + } + + for (Map file : fetcher.getContentStoreFilesDownloaded()) + { + filesDownloaded.add((String) file.get(NAME)); + bytesDownloaded += (Long) file.get(SIZE); + } + + Map currentFile = fetcher.getCurrentFile(); + String currFile = null; + long currFileSize = 0, currFileSizeDownloaded = 0; + float percentDownloaded = 0; + if (currentFile != null) + { + currFile = (String) currentFile.get(NAME); + currFileSize = (Long) currentFile.get(SIZE); + if (currentFile.containsKey("bytesDownloaded")) + { + currFileSizeDownloaded = (Long) currentFile.get("bytesDownloaded"); + bytesDownloaded += currFileSizeDownloaded; + if (currFileSize > 0) + percentDownloaded = (currFileSizeDownloaded * 100) / currFileSize; + } + } + slave.add("filesDownloaded", filesDownloaded); + slave.add("numFilesDownloaded", String.valueOf(filesDownloaded.size())); + + long estimatedTimeRemaining = 0; + + Date replicationStartTimeStamp = fetcher.getReplicationStartTimeStamp(); + if (replicationStartTimeStamp != null) + { + slave.add("replicationStartTime", replicationStartTimeStamp.toString()); + } + long elapsed = fetcher.getReplicationTimeElapsed(); + slave.add("timeElapsed", elapsed + "s"); + + if (bytesDownloaded > 0) + estimatedTimeRemaining = ((bytesToDownload - bytesDownloaded) * elapsed) / bytesDownloaded; + float totalPercent = 0; + long downloadSpeed = 0; + if (bytesToDownload > 0) + totalPercent = (bytesDownloaded * 100) / bytesToDownload; + if (elapsed > 0) + downloadSpeed = (bytesDownloaded / elapsed); + if (currFile != null) + slave.add("currentFile", currFile); + slave.add("currentFileSize", NumberUtils.readableSize(currFileSize)); + slave.add("currentFileSizeDownloaded", NumberUtils.readableSize(currFileSizeDownloaded)); + slave.add("currentFileSizePercent", String.valueOf(percentDownloaded)); + slave.add("bytesDownloaded", NumberUtils.readableSize(bytesDownloaded)); + slave.add("totalPercent", String.valueOf(totalPercent)); + slave.add("timeRemaining", estimatedTimeRemaining + "s"); + slave.add("downloadSpeed", NumberUtils.readableSize(downloadSpeed)); + } + catch (Exception e) + { + LOG.error("Exception while writing replication details: ", e); + } + } + } + + if (isMaster) + details.add("master", master); + if (slave.size() > 0) + details.add("slave", slave); + + NamedList snapshotStats = snapShootDetails; + if (snapshotStats != null) + details.add(CMD_BACKUP, snapshotStats); + + return details; + } + + private void addVal(NamedList nl, String key, Properties props, Class clzz) + { + String s = props.getProperty(key); + if (s == null || s.trim().length() == 0) + return; + if (clzz == Date.class) + { + try + { + long l = Long.parseLong(s); + nl.add(key, new Date(l).toString()); + } + catch (NumberFormatException e) + {/*no op*/ } + } + else if (clzz == List.class) + { + String[] ss = s.split(","); + List l = new ArrayList<>(); + for (String s1 : ss) + { + l.add(new Date(Long.parseLong(s1)).toString()); + } + nl.add(key, l); + } + else + { + nl.add(key, s); + } + + } + + private List getReplicateAfterStrings() + { + List replicateAfter = new ArrayList<>(); + if (replicateOnCommit) + replicateAfter.add("commit"); + if (replicateOnOptimize) + replicateAfter.add("optimize"); + if (replicateOnStart) + replicateAfter.add("startup"); + return replicateAfter; + } + + Properties loadReplicationProperties() + { + Directory dir = null; + try + { + try + { + dir = core.getDirectoryFactory() + .get(core.getDataDir(), DirContext.META_DATA, core.getSolrConfig().indexConfig.lockType); + IndexInput input; + try + { + input = dir.openInput(AlfrescoIndexFetcher.REPLICATION_PROPERTIES, IOContext.DEFAULT); + } + catch (FileNotFoundException | NoSuchFileException e) + { + return new Properties(); + } + + try + { + final InputStream is = new PropertiesInputStream(input); + Properties props = new Properties(); + props.load(new InputStreamReader(is, StandardCharsets.UTF_8)); + return props; + } + finally + { + input.close(); + } + } + finally + { + if (dir != null) + { + core.getDirectoryFactory().release(dir); + } + } + } + catch (IOException e) + { + throw new SolrException(ErrorCode.SERVER_ERROR, e); + } + } + + private void setupPolling(String intervalStr) + { + pollIntervalStr = intervalStr; + pollIntervalNs = readIntervalNs(pollIntervalStr); + if (pollIntervalNs == null || pollIntervalNs <= 0) + { + LOG.info(" No value set for 'pollInterval'. Timer Task not started."); + return; + } + + Runnable task = () -> { + if (pollDisabled.get()) + { + LOG.info("Poll disabled"); + return; + } + try + { + LOG.debug("Polling for index modifications"); + markScheduledExecutionStart(); + doFetch(null, false); + } + catch (Exception e) + { + LOG.error("Exception in fetching index", e); + } + }; + executorService = Executors.newSingleThreadScheduledExecutor(new DefaultSolrThreadFactory("indexFetcher")); + // Randomize initial delay, with a minimum of 1ms + long initialDelayNs = + new Random().nextLong() % pollIntervalNs + TimeUnit.NANOSECONDS.convert(1, TimeUnit.MILLISECONDS); + executorService.scheduleAtFixedRate(task, initialDelayNs, pollIntervalNs, TimeUnit.NANOSECONDS); + LOG.info("Poll scheduled at an interval of {}ms", + TimeUnit.MILLISECONDS.convert(pollIntervalNs, TimeUnit.NANOSECONDS)); + } + + @Override + public void inform(SolrCore core) + { + this.core = core; + + CoreContainer coreContainer = core.getCoreContainer(); + AlfrescoCoreAdminHandler coreAdminHandler = (AlfrescoCoreAdminHandler) coreContainer.getMultiCoreHandler(); + + contentStore = coreAdminHandler.getSolrContentStore(); + registerCloseHook(); + Object nbtk = initArgs.get(NUMBER_BACKUPS_TO_KEEP_INIT_PARAM); + if (nbtk != null) + { + numberBackupsToKeep = Integer.parseInt(nbtk.toString()); + } + else + { + numberBackupsToKeep = 0; + } + NamedList slave = (NamedList) initArgs.get("slave"); + boolean enableSlave = isEnabled(slave); + if (enableSlave) + { + currentAlfrescoIndexFetcher = pollingAlfrescoIndexFetcher = new AlfrescoIndexFetcher(slave, this, core, contentStore); + setupPolling((String) slave.get(POLL_INTERVAL)); + isSlave = true; + } + NamedList master = (NamedList) initArgs.get("master"); + boolean enableMaster = isEnabled(master); + + if (enableMaster || enableSlave) + { + if (core.getCoreContainer().getZkController() != null) + { + LOG.warn("SolrCloud is enabled for core " + core.getName() + + " but so is old-style replication. Make sure you" + + " intend this behavior, it usually indicates a mis-configuration. Master setting is " + + enableMaster + " and slave setting is " + enableSlave); + } + } + + if (!enableSlave && !enableMaster) + { + enableMaster = true; + master = new NamedList<>(); + } + + if (enableMaster) + { + includeConfFiles = (String) master.get(CONF_FILES); + if (includeConfFiles != null && includeConfFiles.trim().length() > 0) + { + String[] files = includeConfFiles.split(","); + for (String file : files) + { + if (file.trim().length() == 0) + continue; + String[] strs = file.trim().split(":"); + // if there is an alias add it or it is null + confFileNameAlias.add(strs[0], strs.length > 1 ? strs[1] : null); + } + LOG.info("Replication enabled for following config files: " + includeConfFiles); + } + List backup = master.getAll("backupAfter"); + boolean backupOnCommit = backup.contains("commit"); + boolean backupOnOptimize = !backupOnCommit && backup.contains("optimize"); + List replicateAfter = master.getAll(REPLICATE_AFTER); + replicateOnCommit = replicateAfter.contains("commit"); + replicateOnOptimize = !replicateOnCommit && replicateAfter.contains("optimize"); + + if (!replicateOnCommit && !replicateOnOptimize) + { + replicateOnCommit = true; + } + + // if we only want to replicate on optimize, we need the deletion policy to + // save the last optimized commit point. + if (replicateOnOptimize) + { + IndexDeletionPolicyWrapper wrapper = core.getDeletionPolicy(); + IndexDeletionPolicy policy = wrapper == null ? null : wrapper.getWrappedDeletionPolicy(); + if (policy instanceof SolrDeletionPolicy) + { + SolrDeletionPolicy solrPolicy = (SolrDeletionPolicy) policy; + if (solrPolicy.getMaxOptimizedCommitsToKeep() < 1) + { + solrPolicy.setMaxOptimizedCommitsToKeep(1); + } + } + else + { + LOG.warn("Replication can't call setMaxOptimizedCommitsToKeep on " + policy); + } + } + + if (replicateOnOptimize || backupOnOptimize) + { + core.getUpdateHandler() + .registerOptimizeCallback(getEventListener(backupOnOptimize, replicateOnOptimize)); + } + if (replicateOnCommit || backupOnCommit) + { + replicateOnCommit = true; + core.getUpdateHandler().registerCommitCallback(getEventListener(backupOnCommit, replicateOnCommit)); + } + if (replicateAfter.contains("startup")) + { + replicateOnStart = true; + RefCounted s = core.getNewestSearcher(false); + try + { + DirectoryReader reader = s == null ? null : s.get().getIndexReader(); + if (reader != null && reader.getIndexCommit() != null + && reader.getIndexCommit().getGeneration() != 1L) + { + // try { + if (replicateOnOptimize) + { + Collection commits = DirectoryReader.listCommits(reader.directory()); + for (IndexCommit ic : commits) + { + if (ic.getSegmentCount() == 1) + { + if (indexCommitPoint == null || indexCommitPoint.getGeneration() < ic + .getGeneration()) + indexCommitPoint = ic; + } + } + } + else + { + indexCommitPoint = reader.getIndexCommit(); + } + } + + // ensure the writer is init'd so that we have a list of commit points + RefCounted iw = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core); + iw.decref(); + + } + catch (IOException e) + { + LOG.warn("Unable to get IndexCommit on startup", e); + } + finally + { + if (s != null) + s.decref(); + } + } + String reserve = (String) master.get(RESERVE); + if (reserve != null && !reserve.trim().equals("")) + { + reserveCommitDuration = readIntervalMs(reserve); + } + LOG.info("Commits will be reserved for " + reserveCommitDuration); + isMaster = true; + } + } + + // check master or slave is enabled + private boolean isEnabled(NamedList params) + { + if (params == null) + return false; + Object enable = params.get("enable"); + if (enable == null) + return true; + if (enable instanceof String) + return StrUtils.parseBool((String) enable); + return Boolean.TRUE.equals(enable); + } + + /** + * register a closehook + */ + private void registerCloseHook() + { + core.addCloseHook(new CloseHook() + { + @Override + public void preClose(SolrCore core) + { + try + { + if (executorService != null) + executorService.shutdown(); // we don't wait for shutdown - this can deadlock core reload + } + finally + { + if (pollingAlfrescoIndexFetcher != null) + { + pollingAlfrescoIndexFetcher.destroy(); + } + } + if (currentAlfrescoIndexFetcher != null && currentAlfrescoIndexFetcher != pollingAlfrescoIndexFetcher) + { + currentAlfrescoIndexFetcher.destroy(); + } + } + + @Override + public void postClose(SolrCore core) + { + } + }); + + core.addCloseHook(new CloseHook() + { + @Override + public void preClose(SolrCore core) + { + ExecutorUtil.shutdownAndAwaitTermination(restoreExecutor); + if (restoreFuture != null) + { + restoreFuture.cancel(false); + } + } + + @Override + public void postClose(SolrCore core) + { + } + }); + } + + /** + * Register a listener for postcommit/optimize + * + * @param snapshoot do a snapshoot + * @param getCommit get a commitpoint also + * @return an instance of the eventlistener + */ + private SolrEventListener getEventListener(final boolean snapshoot, final boolean getCommit) + { + return new SolrEventListener() + { + @Override + public void init(NamedList args) + {/*no op*/ } + + /** + * This refreshes the latest replicateable index commit and optionally can create Snapshots as well + */ + @Override + public void postCommit() + { + IndexCommit currentCommitPoint = core.getDeletionPolicy().getLatestCommit(); + + if (getCommit) + { + // IndexCommit oldCommitPoint = indexCommitPoint; + indexCommitPoint = currentCommitPoint; + } + if (snapshoot) + { + try + { + int numberToKeep = numberBackupsToKeep; + if (numberToKeep < 1) + { + numberToKeep = Integer.MAX_VALUE; + } + + SnapShooter snapShooter = new SnapShooter(core, null, null); + snapShooter.validateCreateSnapshot(); + snapShooter.createSnapAsync(currentCommitPoint, numberToKeep, (nl) -> snapShootDetails = nl); + } + catch (Exception e) + { + LOG.error("Exception while snapshooting", e); + } + } + } + + @Override + public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) + { /*no op*/} + + @Override + public void postSoftCommit() + { + + } + }; + } + + /** + * This class is used to read and send files in the lucene index + */ + private class DirectoryFileStream implements SolrCore.RawWriter + { + protected SolrParams params; + + FastOutputStream fos; + + Long indexGen; + IndexDeletionPolicyWrapper delPolicy; + + String fileName; + String cfileName; + String tlogFileName; + String contentStoreFilename; + String sOffset; + String sLen; + String compress; + boolean useChecksum; + + protected long offset = -1; + int len = -1; + + Checksum checksum; + + private RateLimiter rateLimiter; + + byte[] buf; + + DirectoryFileStream(SolrParams solrParams) + { + params = solrParams; + delPolicy = core.getDeletionPolicy(); + + fileName = validateFilenameOrError(params.get(FILE)); + cfileName = validateFilenameOrError(params.get(CONF_FILE_SHORT)); + tlogFileName = validateFilenameOrError(params.get(TLOG_FILE)); + + sOffset = params.get(OFFSET); + sLen = params.get(LEN); + compress = params.get(COMPRESSION); + useChecksum = params.getBool(CHECKSUM, false); + + indexGen = params.getLong(GENERATION); + if (useChecksum) + { + checksum = new Adler32(); + } + //No throttle if MAX_WRITE_PER_SECOND is not specified + double maxWriteMBPerSec = params.getDouble(MAX_WRITE_PER_SECOND, Double.MAX_VALUE); + rateLimiter = new RateLimiter.SimpleRateLimiter(maxWriteMBPerSec); + } + + // Throw exception on directory traversal attempts + String validateFilenameOrError(String filename) + { + if (filename != null) + { + Path filePath = Paths.get(filename); + filePath.forEach(subpath -> { + if ("..".equals(subpath.toString())) + { + throw new SolrException(ErrorCode.FORBIDDEN, "File name cannot contain .."); + } + }); + if (filePath.isAbsolute()) + { + throw new SolrException(ErrorCode.FORBIDDEN, "File name must be relative"); + } + return filename; + } + else + return null; + } + + void initWrite() throws IOException + { + if (sOffset != null) + offset = Long.parseLong(sOffset); + if (sLen != null) + len = Integer.parseInt(sLen); + if (fileName == null && cfileName == null && tlogFileName == null && contentStoreFilename == null) + { + // no filename do nothing + writeNothingAndFlush(); + } + buf = new byte[(len == -1 || len > PACKET_SZ) ? PACKET_SZ : len]; + + //reserve commit point till write is complete + if (indexGen != null) + { + delPolicy.saveCommitPoint(indexGen); + } + } + + void createOutputStream(OutputStream out) + { + if (Boolean.parseBoolean(compress)) + { + fos = new FastOutputStream(new DeflaterOutputStream(out)); + } + else + { + fos = new FastOutputStream(out); + } + } + + void extendReserveAndReleaseCommitPoint() + { + if (indexGen != null) + { + //Reserve the commit point for another 10s for the next file to be to fetched. + //We need to keep extending the commit reservation between requests so that the replica can fetch + //all the files correctly. + delPolicy.setReserveDuration(indexGen, reserveCommitDuration); + + //release the commit point as the write is complete + delPolicy.releaseCommitPoint(indexGen); + } + + } + + public void write(OutputStream out) throws IOException + { + createOutputStream(out); + + IndexInput in = null; + try + { + initWrite(); + + RefCounted sref = core.getSearcher(); + Directory dir; + try + { + SolrIndexSearcher searcher = sref.get(); + dir = searcher.getIndexReader().directory(); + } + finally + { + sref.decref(); + } + in = dir.openInput(fileName, IOContext.READONCE); + // if offset is mentioned move the pointer to that point + if (offset != -1) + in.seek(offset); + + long filelen = dir.fileLength(fileName); + long maxBytesBeforePause = 0; + + while (true) + { + offset = offset == -1 ? 0 : offset; + int read = (int) Math.min(buf.length, filelen - offset); + in.readBytes(buf, 0, read); + + fos.writeInt(read); + if (useChecksum) + { + checksum.reset(); + checksum.update(buf, 0, read); + fos.writeLong(checksum.getValue()); + } + fos.write(buf, 0, read); + fos.flush(); + LOG.debug("Wrote {} bytes for file {}", offset + read, fileName); + + //Pause if necessary + maxBytesBeforePause += read; + if (maxBytesBeforePause >= rateLimiter.getMinPauseCheckBytes()) + { + rateLimiter.pause(maxBytesBeforePause); + maxBytesBeforePause = 0; + } + if (read != buf.length) + { + writeNothingAndFlush(); + fos.close(); + break; + } + offset += read; + in.seek(offset); + } + } + catch (IOException e) + { + LOG.warn("Exception while writing response for params: " + params, e); + } + finally + { + if (in != null) + { + in.close(); + } + extendReserveAndReleaseCommitPoint(); + } + } + + /** + * Used to write a marker for EOF + */ + protected void writeNothingAndFlush() throws IOException + { + fos.writeInt(0); + fos.flush(); + } + } + + /** + * This is used to read and send files in content store ad a single stream + */ + protected abstract class LocalFsFileStream extends DirectoryFileStream + { + + private File file; + + LocalFsFileStream(SolrParams solrParams) + { + super(solrParams); + this.file = this.initFile(); + } + + protected abstract File initFile(); + + @Override + public void write(OutputStream out) + { + createOutputStream(out); + FileInputStream inputStream = null; + try + { + initWrite(); + + if (file.exists() && file.canRead()) + { + inputStream = new FileInputStream(file); + FileChannel channel = inputStream.getChannel(); + //if offset is mentioned move the pointer to that point + if (offset != -1) + channel.position(offset); + ByteBuffer bb = ByteBuffer.wrap(buf); + + while (true) + { + bb.clear(); + long bytesRead = channel.read(bb); + if (bytesRead <= 0) + { + writeNothingAndFlush(); + fos.close(); + break; + } + fos.writeInt((int) bytesRead); + if (useChecksum) + { + checksum.reset(); + checksum.update(buf, 0, (int) bytesRead); + fos.writeLong(checksum.getValue()); + } + fos.write(buf, 0, (int) bytesRead); + fos.flush(); + } + } + else + { + writeNothingAndFlush(); + } + } + catch (IOException e) + { + LOG.warn("Exception while writing response for params: " + params, e); + } + finally + { + IOUtils.closeQuietly(inputStream); + extendReserveAndReleaseCommitPoint(); + } + } + } + + /** + * This is used to write files in content store. + */ + protected class ContentStoreFilesStream extends DirectoryFileStream + { + ContentStoreFilesStream(SolrParams solrParams) + { + super(solrParams); + } + + @Override + public void write(OutputStream out) throws IOException + { + createOutputStream(out); + String contentStoreRoot = contentStore.getRootLocation(); + try + { + for (String fileName : params.getParams(CONTENT_STORE_FILE_LIST)) + { + File f = new File(contentStoreRoot + fileName); + if (f.exists() && !f.isDirectory()) + { + try + { + writeFile(f, fileName); + } + catch (IOException e) + { + e.printStackTrace(); + } + } + } + } + catch (Exception e) + { + e.printStackTrace(); + } + finally + { + fos.close(); + extendReserveAndReleaseCommitPoint(); + } + } + + @Override + protected void writeNothingAndFlush() throws IOException + { + fos.flush(); + } + + void writeFile(File file, String fileName) throws IOException + { + + buf = new byte[PACKET_SZ]; + if (file.exists() && file.canRead()) + { + FileInputStream inputStream = new FileInputStream(file); + FileChannel channel = inputStream.getChannel(); + //if offset is mentioned move the pointer to that point + if (offset != -1) + channel.position(offset); + ByteBuffer bb = ByteBuffer.wrap(buf); + channel.size(); + + fos.writeInt(fileName.length()); + fos.write(fileName.getBytes()); + fos.writeInt((int) channel.size()); + + if (channel.size() != 0) + { + + while (true) + { + bb.clear(); + long bytesRead = channel.read(bb); + if (bytesRead <= 0) + { + writeNothingAndFlush(); + break; + } + + fos.writeInt((int) bytesRead); + + if (useChecksum) + { + checksum.reset(); + checksum.update(buf, 0, (int) bytesRead); + fos.writeLong(checksum.getValue()); + } + + fos.write(buf, 0, (int) bytesRead); + fos.flush(); + } + } + } + } + } + + private class LocalFsTlogFileStream extends LocalFsFileStream + { + + LocalFsTlogFileStream(SolrParams solrParams) + { + super(solrParams); + } + + protected File initFile() + { + //if it is a tlog file read from tlog directory + return new File(core.getUpdateHandler().getUpdateLog().getLogDir(), tlogFileName); + } + + } + + private class LocalFsConfFileStream extends LocalFsFileStream + { + LocalFsConfFileStream(SolrParams solrParams) + { + super(solrParams); + } + + protected File initFile() + { + //if it is a conf file read from config directory + return new File(core.getResourceLoader().getConfigDir(), cfileName); + } + } + + private static Integer readIntervalMs(String interval) + { + return (int) TimeUnit.MILLISECONDS.convert(readIntervalNs(interval), TimeUnit.NANOSECONDS); + } + + private static Long readIntervalNs(String interval) + { + if (interval == null) + return null; + int result; + Matcher m = INTERVAL_PATTERN.matcher(interval.trim()); + if (m.find()) + { + String hr = m.group(1); + String min = m.group(2); + String sec = m.group(3); + result = 0; + try + { + if (sec != null && sec.length() > 0) + result += Integer.parseInt(sec); + if (min != null && min.length() > 0) + result += (60 * Integer.parseInt(min)); + if (hr != null && hr.length() > 0) + result += (60 * 60 * Integer.parseInt(hr)); + return TimeUnit.NANOSECONDS.convert(result, TimeUnit.SECONDS); + } + catch (NumberFormatException e) + { + throw new SolrException(ErrorCode.SERVER_ERROR, INTERVAL_ERR_MSG); + } + } + else + { + throw new SolrException(ErrorCode.SERVER_ERROR, INTERVAL_ERR_MSG); + } + } + + private static final String SUCCESS = "success"; + + private static final String FAILED = "failed"; + + private static final String EXCEPTION = "exception"; + + static final String MASTER_URL = "masterUrl"; + + private static final String STATUS = "status"; + + static final String COMMAND = "command"; + + static final String CMD_DETAILS = "details"; + + private static final String CMD_BACKUP = "backup"; + + private static final String CMD_RESTORE = "restore"; + + private static final String CMD_RESTORE_STATUS = "restorestatus"; + + private static final String CMD_FETCH_INDEX = "fetchindex"; + + private static final String CMD_ABORT_FETCH = "abortfetch"; + + static final String CMD_GET_FILE_LIST = "filelist"; + + static final String CMD_GET_FILE = "filecontent"; + + private static final String CMD_DISABLE_POLL = "disablepoll"; + + private static final String CMD_DISABLE_REPL = "disablereplication"; + + private static final String CMD_ENABLE_REPL = "enablereplication"; + + private static final String CMD_ENABLE_POLL = "enablepoll"; + + static final String CMD_INDEX_VERSION = "indexversion"; + + private static final String CMD_SHOW_COMMITS = "commits"; + + private static final String CMD_DELETE_BACKUP = "deletebackup"; + + static final String GENERATION = "generation"; + + static final String CONTENT_STORE_VERSION = "contentstoreversion"; + + static final String OFFSET = "offset"; + + private static final String LEN = "len"; + + static final String FILE = "file"; + + public static final String SIZE = "size"; + + private static final String MAX_WRITE_PER_SECOND = "maxWriteMBPerSec"; + + static final String CONF_FILE_SHORT = "cf"; + + static final String TLOG_FILE = "tlogFile"; + + static final String CHECKSUM = "checksum"; + + static final String ALIAS = "alias"; + + static final String CONF_FILES = "confFiles"; + + static final String TLOG_FILES = "tlogFiles"; + + static final String CONTENT_STORE_FILES = "contentStoreFiles"; + + private static final String REPLICATE_AFTER = "replicateAfter"; + + static final String FILE_STREAM = "filestream"; + + private static final String POLL_INTERVAL = "pollInterval"; + + private static final String INTERVAL_ERR_MSG = "The " + POLL_INTERVAL + " must be in this format 'HH:mm:ss'"; + + private static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)"); + + private static final int PACKET_SZ = 1024 * 1024; // 1MB + + private static final String RESERVE = "commitReserveDuration"; + + static final String COMPRESSION = "compression"; + + static final String EXTERNAL = "external"; + + static final String INTERNAL = "internal"; + + private static final String ERR_STATUS = "ERROR"; + + private static final String OK_STATUS = "OK"; + + private static final String NEXT_EXECUTION_AT = "nextExecutionAt"; + + private static final String NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM = "numberToKeep"; + + private static final String NUMBER_BACKUPS_TO_KEEP_INIT_PARAM = "maxNumberOfBackups"; + + static final String CONTENT_STORE_FILE_LIST = "contentStoreFiles"; + + static final String CMD_CONTENT_STORE_FILES = "cmdContentStoreFiles"; + + static final long NO_INDEX_REPLICATION_REQUIRED = -3; + + /** + * Boolean param for tests that can be specified when using + * {@link #CMD_FETCH_INDEX} to force the current request to block until + * the fetch is complete. NOTE: This param is not advised for + * non-test code, since the the duration of the fetch for non-trivial + * indexes will likeley cause the request to time out. + */ + private static final String WAIT = "wait"; +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java index a8da64a57..30db17bf8 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java @@ -1,5 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + * Modification copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see + * + */ package org.alfresco.solr.handler; +import org.apache.solr.handler.SnapShooter; + import java.net.URI; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -10,44 +47,56 @@ import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; -class OldBackupDirectory implements Comparable { - private static final Pattern dirNamePattern = Pattern.compile("^snapshot[.](.*)$"); - private URI basePath; - private String dirName; - private Optional timestamp = Optional.empty(); +class OldBackupDirectory implements Comparable +{ + private static final Pattern dirNamePattern = Pattern.compile("^snapshot[.](.*)$"); - public OldBackupDirectory(URI basePath, String dirName) { - this.dirName = Objects.requireNonNull(dirName); - this.basePath = Objects.requireNonNull(basePath); - Matcher m = dirNamePattern.matcher(dirName); - if (m.find()) { - try { - this.timestamp = Optional.of(new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).parse(m.group(1))); - } catch (ParseException e) { - this.timestamp = Optional.empty(); - } + private URI basePath; + private String dirName; + private Optional timestamp = Optional.empty(); + + public OldBackupDirectory(URI basePath, String dirName) + { + this.dirName = Objects.requireNonNull(dirName); + this.basePath = Objects.requireNonNull(basePath); + Matcher m = dirNamePattern.matcher(dirName); + if (m.find()) + { + try + { + this.timestamp = Optional.of(new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).parse(m.group(1))); + } + catch (ParseException e) + { + this.timestamp = Optional.empty(); + } + } } - } - public URI getPath() { - return this.basePath.resolve(dirName); - } + public URI getPath() + { + return this.basePath.resolve(dirName); + } - public String getDirName() { - return dirName; - } + public String getDirName() + { + return dirName; + } - public Optional getTimestamp() { - return timestamp; - } + public Optional getTimestamp() + { + return timestamp; + } - @Override - public int compareTo(OldBackupDirectory that) { - if(this.timestamp.isPresent() && that.timestamp.isPresent()) { - return that.timestamp.get().compareTo(this.timestamp.get()); + @Override + public int compareTo(OldBackupDirectory that) + { + if (this.timestamp.isPresent() && that.timestamp.isPresent()) + { + return that.timestamp.get().compareTo(this.timestamp.get()); + } + // Use absolute value of path in case the time-stamp is missing on either side. + return that.getPath().compareTo(this.getPath()); } - // Use absolute value of path in case the time-stamp is missing on either side. - return that.getPath().compareTo(this.getPath()); - } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java index d75b94bd1..99652cc71 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java @@ -13,6 +13,25 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * + * Modification copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see + * */ package org.alfresco.solr.handler; @@ -52,237 +71,307 @@ import java.util.function.Consumer; *

Provides functionality equivalent to the snapshooter script

* This is no longer used in standard replication. * - * * @since solr 1.4 */ -public class SnapShooter { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private SolrCore solrCore; - private String snapshotName = null; - private String directoryName = null; - private URI baseSnapDirPath = null; - private URI snapshotDirPath = null; - private BackupRepository backupRepo = null; - private String commitName; // can be null +public class SnapShooter +{ + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private SolrCore solrCore; + private String snapshotName = null; + private String directoryName = null; + private URI baseSnapDirPath = null; + private URI snapshotDirPath = null; + private BackupRepository backupRepo = null; + private String commitName; // can be null - @Deprecated - public SnapShooter(SolrCore core, String location, String snapshotName) { - String snapDirStr = null; - // Note - This logic is only applicable to the usecase where a shared file-system is exposed via - // local file-system interface (primarily for backwards compatibility). For other use-cases, users - // will be required to specify "location" where the backup should be stored. - if (location == null) { - snapDirStr = core.getDataDir(); - } else { - snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString(); - } - initialize(new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null); - } - - public SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) { - initialize(backupRepo, core, location, snapshotName, commitName); - } - - private void initialize(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) { - this.solrCore = Objects.requireNonNull(core); - this.backupRepo = Objects.requireNonNull(backupRepo); - this.baseSnapDirPath = location; - this.snapshotName = snapshotName; - if (snapshotName != null) { - directoryName = "snapshot." + snapshotName; - } else { - SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.ROOT); - directoryName = "snapshot." + fmt.format(new Date()); - } - this.snapshotDirPath = backupRepo.resolve(location, directoryName); - this.commitName = commitName; - } - - public BackupRepository getBackupRepository() { - return backupRepo; - } - - /** - * Gets the parent directory of the snapshots. This is the {@code location} - * given in the constructor. - */ - public URI getLocation() { - return this.baseSnapDirPath; - } - - public void validateDeleteSnapshot() { - Objects.requireNonNull(this.snapshotName); - - boolean dirFound = false; - String[] paths; - try { - paths = backupRepo.listAll(baseSnapDirPath); - for (String path : paths) { - if (path.equals(this.directoryName) - && backupRepo.getPathType(baseSnapDirPath.resolve(path)) == PathType.DIRECTORY) { - dirFound = true; - break; + @Deprecated + public SnapShooter(SolrCore core, String location, String snapshotName) + { + String snapDirStr = null; + // Note - This logic is only applicable to the usecase where a shared file-system is exposed via + // local file-system interface (primarily for backwards compatibility). For other use-cases, users + // will be required to specify "location" where the backup should be stored. + if (location == null) + { + snapDirStr = core.getDataDir(); } - } - if(dirFound == false) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Snapshot " + snapshotName + " cannot be found in directory: " + baseSnapDirPath); - } - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to find snapshot " + snapshotName + " in directory: " + baseSnapDirPath, e); - } - } - - protected void deleteSnapAsync(final ReplicationHandler replicationHandler) { - new Thread(() -> deleteNamedSnapshot(replicationHandler)).start(); - } - - public void validateCreateSnapshot() throws IOException { - // Note - Removed the current behavior of creating the directory hierarchy. - // Do we really need to provide this support? - if (!backupRepo.exists(baseSnapDirPath)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - " Directory does not exist: " + snapshotDirPath); - } - - if (backupRepo.exists(snapshotDirPath)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Snapshot directory already exists: " + snapshotDirPath); - } - } - - public NamedList createSnapshot() throws Exception { - RefCounted searcher = solrCore.getSearcher(); - try { - if (commitName != null) { - SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager(); - Optional commit = snapshotMgr.getIndexCommitByName(commitName); - if(commit.isPresent()) { - return createSnapshot(commit.get()); + else + { + snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString(); } - throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to find an index commit with name " + commitName + - " for core " + solrCore.getName()); - } else { - //TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first? - IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy(); - IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit(); - deletionPolicy.saveCommitPoint(indexCommit.getGeneration()); - try { - return createSnapshot(indexCommit); - } finally { - deletionPolicy.releaseCommitPoint(indexCommit.getGeneration()); + initialize(new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null); + } + + public SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) + { + initialize(backupRepo, core, location, snapshotName, commitName); + } + + private void initialize(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) + { + this.solrCore = Objects.requireNonNull(core); + this.backupRepo = Objects.requireNonNull(backupRepo); + this.baseSnapDirPath = location; + this.snapshotName = snapshotName; + if (snapshotName != null) + { + directoryName = "snapshot." + snapshotName; } - } - } finally { - searcher.decref(); - } - } - - public void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, Consumer result) { - solrCore.getDeletionPolicy().saveCommitPoint(indexCommit.getGeneration()); - - //TODO should use Solr's ExecutorUtil - new Thread(() -> { - try { - result.accept(createSnapshot(indexCommit)); - } catch (Exception e) { - LOG.error("Exception while creating snapshot", e); - NamedList snapShootDetails = new NamedList<>(); - snapShootDetails.add("snapShootException", e.getMessage()); - result.accept(snapShootDetails); - } finally { - solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration()); - } - if (snapshotName == null) { - try { - deleteOldBackups(numberToKeep); - } catch (IOException e) { - LOG.warn("Unable to delete old snapshots ", e); + else + { + SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.ROOT); + directoryName = "snapshot." + fmt.format(new Date()); } - } - }).start(); + this.snapshotDirPath = backupRepo.resolve(location, directoryName); + this.commitName = commitName; + } - } + public BackupRepository getBackupRepository() + { + return backupRepo; + } - // note: remember to reserve the indexCommit first so it won't get deleted concurrently - protected NamedList createSnapshot(final IndexCommit indexCommit) throws Exception { - LOG.info("Creating backup snapshot " + (snapshotName == null ? "" : snapshotName) + " at " + baseSnapDirPath); - boolean success = false; - try { - NamedList details = new NamedList<>(); - details.add("startTime", new Date().toString());//bad; should be Instant.now().toString() + /** + * Gets the parent directory of the snapshots. This is the {@code location} + * given in the constructor. + */ + public URI getLocation() + { + return this.baseSnapDirPath; + } - Collection files = indexCommit.getFileNames(); - Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); - try { - for(String fileName : files) { - backupRepo.copyFileFrom(dir, fileName, snapshotDirPath); + public void validateDeleteSnapshot() + { + Objects.requireNonNull(this.snapshotName); + + boolean dirFound = false; + String[] paths; + try + { + paths = backupRepo.listAll(baseSnapDirPath); + for (String path : paths) + { + if (path.equals(this.directoryName) + && backupRepo.getPathType(baseSnapDirPath.resolve(path)) == PathType.DIRECTORY) + { + dirFound = true; + break; + } + } + if (dirFound == false) + { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Snapshot " + snapshotName + " cannot be found in directory: " + baseSnapDirPath); + } } - } finally { - solrCore.getDirectoryFactory().release(dir); - } - - details.add("fileCount", files.size()); - details.add("status", "success"); - details.add("snapshotCompletedAt", new Date().toString());//bad; should be Instant.now().toString() - details.add("snapshotName", snapshotName); - LOG.info("Done creating backup snapshot: " + (snapshotName == null ? "" : snapshotName) + - " at " + baseSnapDirPath); - success = true; - return details; - } finally { - if (!success) { - try { - backupRepo.deleteDirectory(snapshotDirPath); - } catch (Exception excDuringDelete) { - LOG.warn("Failed to delete "+snapshotDirPath+" after snapshot creation failed due to: "+excDuringDelete); + catch (IOException e) + { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unable to find snapshot " + snapshotName + " in directory: " + baseSnapDirPath, e); } - } } - } - private void deleteOldBackups(int numberToKeep) throws IOException { - String[] paths = backupRepo.listAll(baseSnapDirPath); - List dirs = new ArrayList<>(); - for (String f : paths) { - if (backupRepo.getPathType(baseSnapDirPath.resolve(f)) == PathType.DIRECTORY) { - OldBackupDirectory obd = new OldBackupDirectory(baseSnapDirPath, f); - if (obd.getTimestamp().isPresent()) { - dirs.add(obd); + protected void deleteSnapAsync(final AlfrescoReplicationHandler alfrescoReplicationHandler) + { + new Thread(() -> deleteNamedSnapshot(alfrescoReplicationHandler)).start(); + } + + public void validateCreateSnapshot() throws IOException + { + // Note - Removed the current behavior of creating the directory hierarchy. + // Do we really need to provide this support? + if (!backupRepo.exists(baseSnapDirPath)) + { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + " Directory does not exist: " + snapshotDirPath); } - } - } - if (numberToKeep > dirs.size() -1) { - return; - } - Collections.sort(dirs); - int i=1; - for (OldBackupDirectory dir : dirs) { - if (i++ > numberToKeep) { - backupRepo.deleteDirectory(dir.getPath()); - } - } - } - protected void deleteNamedSnapshot(ReplicationHandler replicationHandler) { - LOG.info("Deleting snapshot: " + snapshotName); - - NamedList details = new NamedList<>(); - - try { - URI path = baseSnapDirPath.resolve("snapshot." + snapshotName); - backupRepo.deleteDirectory(path); - - details.add("status", "success"); - details.add("snapshotDeletedAt", new Date().toString()); - - } catch (IOException e) { - details.add("status", "Unable to delete snapshot: " + snapshotName); - LOG.warn("Unable to delete snapshot: " + snapshotName, e); + if (backupRepo.exists(snapshotDirPath)) + { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Snapshot directory already exists: " + snapshotDirPath); + } } - replicationHandler.snapShootDetails = details; - } + public NamedList createSnapshot() throws Exception + { + RefCounted searcher = solrCore.getSearcher(); + try + { + if (commitName != null) + { + SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager(); + Optional commit = snapshotMgr.getIndexCommitByName(commitName); + if (commit.isPresent()) + { + return createSnapshot(commit.get()); + } + throw new SolrException(ErrorCode.SERVER_ERROR, + "Unable to find an index commit with name " + commitName + " for core " + solrCore.getName()); + } + else + { + //TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first? + IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy(); + IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit(); + deletionPolicy.saveCommitPoint(indexCommit.getGeneration()); + try + { + return createSnapshot(indexCommit); + } + finally + { + deletionPolicy.releaseCommitPoint(indexCommit.getGeneration()); + } + } + } + finally + { + searcher.decref(); + } + } - public static final String DATE_FMT = "yyyyMMddHHmmssSSS"; + public void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, Consumer result) + { + solrCore.getDeletionPolicy().saveCommitPoint(indexCommit.getGeneration()); + + new Thread(() -> { + try + { + result.accept(createSnapshot(indexCommit)); + } + catch (Exception e) + { + LOG.error("Exception while creating snapshot", e); + NamedList snapShootDetails = new NamedList<>(); + snapShootDetails.add("snapShootException", e.getMessage()); + result.accept(snapShootDetails); + } + finally + { + solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration()); + } + if (snapshotName == null) + { + try + { + deleteOldBackups(numberToKeep); + } + catch (IOException e) + { + LOG.warn("Unable to delete old snapshots ", e); + } + } + }).start(); + + } + + // note: remember to reserve the indexCommit first so it won't get deleted concurrently + protected NamedList createSnapshot(final IndexCommit indexCommit) throws Exception + { + LOG.info("Creating backup snapshot " + (snapshotName == null ? "" : snapshotName) + " at " + + baseSnapDirPath); + boolean success = false; + try + { + NamedList details = new NamedList<>(); + details.add("startTime", new Date().toString());//bad; should be Instant.now().toString() + + Collection files = indexCommit.getFileNames(); + Directory dir = solrCore.getDirectoryFactory() + .get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); + try + { + for (String fileName : files) + { + backupRepo.copyFileFrom(dir, fileName, snapshotDirPath); + } + } + finally + { + solrCore.getDirectoryFactory().release(dir); + } + + details.add("fileCount", files.size()); + details.add("status", "success"); + details.add("snapshotCompletedAt", new Date().toString());//bad; should be Instant.now().toString() + details.add("snapshotName", snapshotName); + LOG.info("Done creating backup snapshot: " + (snapshotName == null ? "" : snapshotName) + " at " + + baseSnapDirPath); + success = true; + return details; + } + finally + { + if (!success) + { + try + { + backupRepo.deleteDirectory(snapshotDirPath); + } + catch (Exception excDuringDelete) + { + LOG.warn("Failed to delete " + snapshotDirPath + " after snapshot creation failed due to: " + + excDuringDelete); + } + } + } + } + + private void deleteOldBackups(int numberToKeep) throws IOException + { + String[] paths = backupRepo.listAll(baseSnapDirPath); + List dirs = new ArrayList<>(); + for (String f : paths) + { + if (backupRepo.getPathType(baseSnapDirPath.resolve(f)) == PathType.DIRECTORY) + { + OldBackupDirectory obd = new OldBackupDirectory(baseSnapDirPath, f); + if (obd.getTimestamp().isPresent()) + { + dirs.add(obd); + } + } + } + if (numberToKeep > dirs.size() - 1) + { + return; + } + Collections.sort(dirs); + int i = 1; + for (OldBackupDirectory dir : dirs) + { + if (i++ > numberToKeep) + { + backupRepo.deleteDirectory(dir.getPath()); + } + } + } + + protected void deleteNamedSnapshot(AlfrescoReplicationHandler alfrescoReplicationHandler) + { + LOG.info("Deleting snapshot: " + snapshotName); + + NamedList details = new NamedList<>(); + + try + { + URI path = baseSnapDirPath.resolve("snapshot." + snapshotName); + backupRepo.deleteDirectory(path); + + details.add("status", "success"); + details.add("snapshotDeletedAt", new Date().toString()); + + } + catch (IOException e) + { + details.add("status", "Unable to delete snapshot: " + snapshotName); + LOG.warn("Unable to delete snapshot: " + snapshotName, e); + } + + alfrescoReplicationHandler.snapShootDetails = details; + } + + public static final String DATE_FMT = "yyyyMMddHHmmssSSS"; } diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml index a65aefcfe..cd99467e1 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml @@ -189,7 +189,7 @@ class="solr.XMLResponseWriter" /> - + commit schema.xml diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml index 790e1af2c..8d92bca34 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml @@ -188,7 +188,7 @@ - + {masterURL} 00:00:02 From ba70cbd4773413c05df645830c7a586847d5d6d3 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 1 Nov 2019 11:13:09 +0100 Subject: [PATCH 44/76] [contentStoreReplication] code refactoring --- .../alfresco/solr/handler/IndexFetcher.java | 2323 ----------------- .../solr/handler/ReplicationHandler.java | 1955 -------------- 2 files changed, 4278 deletions(-) delete mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/IndexFetcher.java delete mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/IndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/IndexFetcher.java deleted file mode 100644 index 6113b2343..000000000 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/IndexFetcher.java +++ /dev/null @@ -1,2323 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.alfresco.solr.handler; - -import static java.util.List.of; -import static org.alfresco.solr.handler.ReplicationHandler.ALIAS; -import static org.alfresco.solr.handler.ReplicationHandler.CHECKSUM; -import static org.alfresco.solr.handler.ReplicationHandler.CMD_CONTENT_STORE_FILES; -import static org.alfresco.solr.handler.ReplicationHandler.CMD_DETAILS; -import static org.alfresco.solr.handler.ReplicationHandler.CMD_GET_FILE; -import static org.alfresco.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST; -import static org.alfresco.solr.handler.ReplicationHandler.CMD_INDEX_VERSION; -import static org.alfresco.solr.handler.ReplicationHandler.COMMAND; -import static org.alfresco.solr.handler.ReplicationHandler.COMPRESSION; -import static org.alfresco.solr.handler.ReplicationHandler.CONF_FILES; -import static org.alfresco.solr.handler.ReplicationHandler.CONF_FILE_SHORT; -import static org.alfresco.solr.handler.ReplicationHandler.CONTENT_STORE_FILES; -import static org.alfresco.solr.handler.ReplicationHandler.CONTENT_STORE_FILE_LIST; -import static org.alfresco.solr.handler.ReplicationHandler.CONTENT_STORE_VERSION; -import static org.alfresco.solr.handler.ReplicationHandler.EXTERNAL; -import static org.alfresco.solr.handler.ReplicationHandler.FILE; -import static org.alfresco.solr.handler.ReplicationHandler.FILE_STREAM; -import static org.alfresco.solr.handler.ReplicationHandler.FileInfo; -import static org.alfresco.solr.handler.ReplicationHandler.GENERATION; -import static org.alfresco.solr.handler.ReplicationHandler.INTERNAL; -import static org.alfresco.solr.handler.ReplicationHandler.MASTER_URL; -import static org.alfresco.solr.handler.ReplicationHandler.NO_INDEX_REPLICATION_REQUIRED; -import static org.alfresco.solr.handler.ReplicationHandler.OFFSET; -import static org.alfresco.solr.handler.ReplicationHandler.SIZE; -import static org.alfresco.solr.handler.ReplicationHandler.TLOG_FILE; -import static org.alfresco.solr.handler.ReplicationHandler.TLOG_FILES; -import static org.alfresco.solr.handler.ReplicationHandler.getCheckSum; -import static org.apache.solr.common.params.CommonParams.JAVABIN; -import static org.apache.solr.common.params.CommonParams.NAME; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import org.alfresco.solr.content.SolrContentStore; -import org.apache.http.client.HttpClient; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.FastInputStream; -import org.apache.solr.common.util.IOUtils; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SuppressForbidden; -import org.apache.solr.core.DirectoryFactory; -import org.apache.solr.core.DirectoryFactory.DirContext; -import org.apache.solr.core.IndexDeletionPolicyWrapper; -import org.apache.solr.core.SolrCore; -import org.apache.solr.handler.SnapShooter; -import org.apache.solr.request.LocalSolrQueryRequest; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.update.CdcrUpdateLog; -import org.apache.solr.update.CommitUpdateCommand; -import org.apache.solr.update.UpdateLog; -import org.apache.solr.update.VersionInfo; -import org.apache.solr.util.DefaultSolrThreadFactory; -import org.apache.solr.util.FileUtils; -import org.apache.solr.util.PropertiesOutputStream; -import org.apache.solr.util.RTimer; -import org.apache.solr.util.RefCounted; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.EOFException; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.lang.invoke.MethodHandles; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import java.util.zip.Adler32; -import java.util.zip.Checksum; -import java.util.zip.InflaterInputStream; - -/** - *

Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the - * master.

- * - * @since solr 1.4 - */ -class IndexFetcher { - private static final int _100K = 100000; - - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final int CONTENT_STORE_PARTITION_SIZE = 50; - - private final String masterUrl; - - private final ReplicationHandler replicationHandler; - private final SolrContentStore contentStore; - - private volatile Date replicationStartTimeStamp; - private RTimer replicationTimer; - - private final SolrCore solrCore; - - private volatile List> filesToDownload; - - private volatile List> confFilesToDownload; - - private volatile List> tlogFilesToDownload; - - private volatile List> contentStoreFilesToDownload; - - private volatile List> contentStoreFilesToDelete; - - private volatile List> filesDownloaded; - - private volatile List> confFilesDownloaded; - - private volatile List> tlogFilesDownloaded; - - private volatile List> contentStoreFilesDownloaded; - - private volatile Map currentFile; - - private volatile DirectoryFileFetcher dirFileFetcher; - - private volatile LocalFsFileFetcher localFileFetcher; - - private volatile ContentStoreFetcher contentStoreFileFetcher; - - private volatile ExecutorService fsyncService; - - private volatile boolean stop = false; - - private boolean useInternalCompression; - - private boolean useExternalCompression; - - private final HttpClient myHttpClient; - - private static final String INTERRUPT_RESPONSE_MESSAGE = "Interrupted while waiting for modify lock"; - - private boolean fullContentStoreReplication = false; - - - public static class IndexFetchResult { - private final String message; - private final boolean successful; - private final Throwable exception; - - static final String FAILED_BY_INTERRUPT_MESSAGE = "Fetching index failed by interrupt"; - static final String FAILED_BY_EXCEPTION_MESSAGE = "Fetching index failed by exception"; - - /** pre-defined results */ - static final IndexFetchResult ALREADY_IN_SYNC = new IndexFetchResult("Local index commit is already in sync with peer", true, null); - static final IndexFetchResult INDEX_FETCH_FAILURE = new IndexFetchResult("Fetching lastest index is failed", false, null); - static final IndexFetchResult INDEX_FETCH_SUCCESS = new IndexFetchResult("Fetching latest index is successful", true, null); - static final IndexFetchResult LOCK_OBTAIN_FAILED = new IndexFetchResult("Obtaining SnapPuller lock failed", false, null); - static final IndexFetchResult MASTER_VERSION_ZERO = new IndexFetchResult("Index in peer is empty and never committed yet", true, null); - static final IndexFetchResult NO_INDEX_COMMIT_EXIST = new IndexFetchResult("No IndexCommit in local index", false, null); - static final IndexFetchResult PEER_INDEX_COMMIT_DELETED = new IndexFetchResult("No files to download because IndexCommit in peer was deleted", false, null); - - IndexFetchResult(String message, boolean successful, Throwable exception) { - this.message = message; - this.successful = successful; - this.exception = exception; - } - - /* - * @return exception thrown if failed by exception or interrupt, otherwise null - */ - public Throwable getException() { - return this.exception; - } - - /* - * @return true if index fetch was successful, false otherwise - */ - boolean getSuccessful() { - return this.successful; - } - - public String getMessage() { - return this.message; - } - } - - private static HttpClient createHttpClient(SolrCore core, String connTimeout, String readTimeout, String httpBasicAuthUser, String httpBasicAuthPassword, boolean useCompression) { - final ModifiableSolrParams httpClientParams = new ModifiableSolrParams(); - httpClientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connTimeout != null ? connTimeout : "5000"); - httpClientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, readTimeout != null ? readTimeout : "20000"); - httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_USER, httpBasicAuthUser); - httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword); - httpClientParams.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, useCompression); - - return HttpClientUtil.createClient(httpClientParams, core.getCoreContainer().getUpdateShardHandler().getConnectionManager()); - } - - IndexFetcher(final NamedList initArgs, final ReplicationHandler handler, final SolrCore sc, SolrContentStore contentStore) { - this.contentStore = contentStore; - solrCore = sc; - String masterUrl = (String) initArgs.get(MASTER_URL); - if (masterUrl == null) - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "'masterUrl' is required for a slave"); - if (masterUrl.endsWith(ReplicationHandler.PATH)) { - masterUrl = masterUrl.substring(0, masterUrl.length()-12); - LOG.warn("'masterUrl' must be specified without the "+ ReplicationHandler.PATH+" suffix"); - } - this.masterUrl = masterUrl; - - this.replicationHandler = handler; - String compress = (String) initArgs.get(COMPRESSION); - useInternalCompression = INTERNAL.equals(compress); - useExternalCompression = EXTERNAL.equals(compress); - String connTimeout = (String) initArgs.get(HttpClientUtil.PROP_CONNECTION_TIMEOUT); - // allow a master override for tests - you specify this in /replication slave section of solrconfig and some - // test don't want to define this - String readTimeout = System.getProperty("solr.indexfetcher.sotimeout", "-1"); - if (readTimeout.equals("-1")) { - readTimeout = (String) initArgs.get(HttpClientUtil.PROP_SO_TIMEOUT); - } - String httpBasicAuthUser = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_USER); - String httpBasicAuthPassword = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_PASS); - myHttpClient = createHttpClient(solrCore, connTimeout, readTimeout, httpBasicAuthUser, httpBasicAuthPassword, useExternalCompression); - } - - /** - * Gets the latest commit version and generation from the master - */ - private NamedList getLatestVersion() throws IOException { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(COMMAND, CMD_INDEX_VERSION); - params.set(CommonParams.WT, JAVABIN); - params.set(CommonParams.QT, ReplicationHandler.PATH); - QueryRequest req = new QueryRequest(params); - - // TODO modify to use shardhandler - try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) { - client.setSoTimeout(60000); - client.setConnectionTimeout(15000); - - return client.request(req); - } catch (SolrServerException e) { - throw new SolrException(ErrorCode.SERVER_ERROR, e.getMessage(), e); - } - } - - /** - * Fetches the list of files in a given index commit point and updates internal list of files to download. - */ - @SuppressWarnings("unchecked") - private void fetchFileList(long indexGeneration, long contentStoreGeneration) throws IOException { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(COMMAND, CMD_GET_FILE_LIST); - params.set(GENERATION, String.valueOf(indexGeneration)); - params.set(CONTENT_STORE_VERSION, String.valueOf(contentStoreGeneration)); - params.set(CommonParams.WT, JAVABIN); - params.set(CommonParams.QT, ReplicationHandler.PATH); - QueryRequest req = new QueryRequest(params); - - try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) { - client.setSoTimeout(60000); - client.setConnectionTimeout(15000); - NamedList response = client.request(req); - - List> files = (List>) response.get(CMD_GET_FILE_LIST); - if (files != null) - filesToDownload = Collections.synchronizedList(files); - else { - filesToDownload = Collections.emptyList(); - LOG.info("No files to download for index generation: "+ indexGeneration); - } - - files = (List>) response.get(CONF_FILES); - if (files != null) { - confFilesToDownload = Collections.synchronizedList(files); - } - - files = (List>) response.get(TLOG_FILES); - if (files != null) { - tlogFilesToDownload = Collections.synchronizedList(files); - } - Map>> contentStoreMap = - (Map > >) response.get(CONTENT_STORE_FILES); - - if (contentStoreMap != null) { - contentStoreFilesToDownload = Collections.synchronizedList(contentStoreMap.get(SolrContentStore.ADDS)); - contentStoreFilesToDelete = Collections.synchronizedList(contentStoreMap.get(SolrContentStore.DELETES)); - - List> infomap = contentStoreMap.get(SolrContentStore.INFO); - fullContentStoreReplication = of(infomap).stream() - .flatMap(List::stream) - .map(e -> e.get(SolrContentStore.FULL_REPLICATION)) - .map(e -> Boolean.class.isInstance(e)? (Boolean) e : false) - .findFirst() - .orElse(false); - - } - - } catch (SolrServerException e) { - throw new IOException(e); - } - } - - IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean replicateContentStore) throws IOException, InterruptedException { - return fetchLatestIndex(forceReplication, false, replicateContentStore); - } - - /** - * This command downloads all the necessary files from master to install a index commit point. Only changed files are - * downloaded. It also downloads the conf files (if they are modified). - * - * @param forceReplication force a replication in all cases - * @param forceCoreReload force a core reload in all cases - * @return true on success, false if slave is already in sync - * @throws IOException if an exception occurs - */ - private IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreReload, boolean replicateContentStore) - throws IOException, InterruptedException { - - boolean cleanupDone = false; - boolean successfulInstall = false; - markReplicationStart(); - Directory tmpIndexDir = null; - String tmpIndex; - Directory indexDir = null; - String indexDirPath; - boolean deleteTmpIdxDir = true; - File tmpTlogDir = null; - - if (!solrCore.getSolrCoreState().getLastReplicateIndexSuccess()) { - // if the last replication was not a success, we force a full replication - // when we are a bit more confident we may want to try a partial replication - // if the error is connection related or something, but we have to be careful - forceReplication = true; - } - - try { - //get the current 'replicateable' index version in the master - NamedList response; - try { - response = getLatestVersion(); - } catch (Exception e) { - final String errorMsg = e.toString(); - if (!Strings.isNullOrEmpty(errorMsg) && errorMsg.contains(INTERRUPT_RESPONSE_MESSAGE)) { - LOG.warn("Master at: " + masterUrl + " is not available. Index fetch failed by interrupt. Exception: " + errorMsg); - return new IndexFetchResult(IndexFetchResult.FAILED_BY_INTERRUPT_MESSAGE, false, e); - } else { - LOG.warn("Master at: " + masterUrl + " is not available. Index fetch failed by exception: " + errorMsg); - return new IndexFetchResult(IndexFetchResult.FAILED_BY_EXCEPTION_MESSAGE, false, e); - } - } - - long latestVersion = (Long) response.get(CMD_INDEX_VERSION); - long masterContentStoreVersion = (Long) response.get(CONTENT_STORE_VERSION); - long latestGeneration = (Long) response.get(GENERATION); - - - // The following session should make sure that if replication is happening in more cores at the same time, - // the contentStore replication is done only once. - long slaveContentStoreVersion = replicateContentStore? contentStore.getLastCommittedVersion() : SolrContentStore.NO_CONTENT_STORE_REPLICATION_REQUIRED; - boolean contentStoreReplicationNeeded = replicateContentStore && (masterContentStoreVersion != slaveContentStoreVersion); - boolean indexReplicationNeeded = true; - - - LOG.info("Master's generation: " + latestGeneration); - LOG.info("Master's version: " + latestVersion); - - // TODO: make sure that getLatestCommit only returns commit points for the main index (i.e. no side-car indexes) - IndexCommit commit = solrCore.getDeletionPolicy().getLatestCommit(); - if (commit == null) { - // Presumably the IndexWriter hasn't been opened yet, and hence the deletion policy hasn't been updated with commit points - RefCounted searcherRefCounted = null; - try { - searcherRefCounted = solrCore.getNewestSearcher(false); - if (searcherRefCounted == null) { - LOG.warn("No open searcher found - fetch aborted"); - return IndexFetchResult.NO_INDEX_COMMIT_EXIST; - } - commit = searcherRefCounted.get().getIndexReader().getIndexCommit(); - } finally { - if (searcherRefCounted != null) - searcherRefCounted.decref(); - } - } - - LOG.info("Slave's generation: " + commit.getGeneration()); - LOG.info("Slave's version: " + IndexDeletionPolicyWrapper.getCommitTimestamp(commit)); - - if (latestVersion == 0L) { - if (forceReplication && commit.getGeneration() != 0) { - // since we won't get the files for an empty index, - // we just clear ours and commit - RefCounted iw = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore); - try { - iw.get().deleteAll(); - } finally { - iw.decref(); - } - SolrQueryRequest req = new LocalSolrQueryRequest(solrCore, new ModifiableSolrParams()); - solrCore.getUpdateHandler().commit(new CommitUpdateCommand(req, false)); - } - - //there is nothing to be replicated - successfulInstall = true; - return IndexFetchResult.MASTER_VERSION_ZERO; - } - - - - // TODO: Should we be comparing timestamps (across machines) here? - if (!forceReplication && IndexDeletionPolicyWrapper.getCommitTimestamp(commit) == latestVersion) { - //master and slave are already in sync just return - LOG.info("Slave index in sync with master."); - successfulInstall = true; - indexReplicationNeeded = false; - } - - if (!indexReplicationNeeded && !contentStoreReplicationNeeded){ - return IndexFetchResult.ALREADY_IN_SYNC; - } - - if (indexReplicationNeeded) - LOG.info("Starting index replication process"); - - if (contentStoreReplicationNeeded) - LOG.info("Starting content store replication process"); - - if (masterContentStoreVersion < slaveContentStoreVersion) - { - LOG.error("slave content store version is not valid. Full content store replication required"); - slaveContentStoreVersion = SolrContentStore.NO_VERSION_AVAILABLE; - } - - - // get the list of files first - fetchFileList(indexReplicationNeeded? latestGeneration : NO_INDEX_REPLICATION_REQUIRED, slaveContentStoreVersion); - // this can happen if the commit point is deleted before we fetch the file list. - if (filesToDownload.isEmpty() && indexReplicationNeeded) { - return IndexFetchResult.PEER_INDEX_COMMIT_DELETED; - } - - if (indexReplicationNeeded) - { - - LOG.info("Number of files in latest index in master: " + filesToDownload.size()); - if (tlogFilesToDownload != null) { - LOG.info("Number of tlog files in master: " + tlogFilesToDownload.size()); - } - } - - // Create the sync service - fsyncService = ExecutorUtil.newMDCAwareSingleThreadExecutor(new DefaultSolrThreadFactory("fsyncService")); - // use a synchronized list because the list is read by other threads (to show details) - filesDownloaded = Collections.synchronizedList(new ArrayList<>()); - - // if the generation of master is older than that of the slave , it means they are not compatible to be copied - // then a new index directory to be created and all the files need to be copied - boolean isFullCopyNeeded = indexReplicationNeeded && (IndexDeletionPolicyWrapper - .getCommitTimestamp(commit) >= latestVersion - || commit.getGeneration() >= latestGeneration || forceReplication); - - String timestamp = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date()); - String tmpIdxDirName = "index." + timestamp; - tmpIndex = solrCore.getDataDir() + tmpIdxDirName; - - tmpIndexDir = solrCore.getDirectoryFactory().get(tmpIndex, DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); - - // tmp dir for tlog files - if (tlogFilesToDownload != null) { - tmpTlogDir = new File(solrCore.getUpdateHandler().getUpdateLog().getLogDir(), "tlog." + timestamp); - } - - // cindex dir... - indexDirPath = solrCore.getIndexDir(); - indexDir = solrCore.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); - - try { - - if (indexReplicationNeeded) - { - - //We will compare all the index files from the master vs the index files on disk to see if there is a mismatch - //in the metadata. If there is a mismatch for the same index file then we download the entire index again. - if (!isFullCopyNeeded && isIndexStale(indexDir)) { - isFullCopyNeeded = true; - } - - if (!isFullCopyNeeded) { - // a searcher might be using some flushed but not committed segments - // because of soft commits (which open a searcher on IW's data) - // so we need to close the existing searcher on the last commit - // and wait until we are able to clean up all unused lucene files - if (solrCore.getCoreContainer().isZooKeeperAware()) { - solrCore.closeSearcher(); - } - - // rollback and reopen index writer and wait until all unused files - // are successfully deleted - solrCore.getUpdateHandler().newIndexWriter(true); - RefCounted writer = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null); - try { - IndexWriter indexWriter = writer.get(); - int c = 0; - indexWriter.deleteUnusedFiles(); - while (hasUnusedFiles(indexDir, commit)) { - indexWriter.deleteUnusedFiles(); - LOG.info("Sleeping for 1000ms to wait for unused lucene index files to be delete-able"); - Thread.sleep(1000); - c++; - if (c >= 30) { - LOG.warn("IndexFetcher unable to cleanup unused lucene index files so we must do a full copy instead"); - isFullCopyNeeded = true; - break; - } - } - if (c > 0) { - LOG.info("IndexFetcher slept for " + (c * 1000) + "ms for unused lucene index files to be delete-able"); - } - } finally { - writer.decref(); - } - } - } - - boolean reloadCore = false; - - try { - // we have to be careful and do this after we know isFullCopyNeeded won't be flipped - if (!isFullCopyNeeded) { - solrCore.getUpdateHandler().getSolrCoreState().closeIndexWriter(solrCore, true); - } - - LOG.info("Starting download (fullCopy={}) to {}", isFullCopyNeeded, tmpIndexDir); - successfulInstall = false; - - long bytesDownloaded = 0; - - if (indexReplicationNeeded){ - downloadIndexFiles(isFullCopyNeeded, indexDir, tmpIndexDir, latestGeneration); - - if (tlogFilesToDownload != null) { - bytesDownloaded += downloadTlogFiles(tmpTlogDir, latestGeneration); - reloadCore = true; // reload update log - } - } - - - if (contentStoreReplicationNeeded) - { - - if (contentStoreFilesToDownload != null){ - bytesDownloaded += downloadContentStoreFiles(contentStore.getRootLocation()); - } - - if (contentStoreFilesToDelete != null) - { - deleteContentStoreFiles(contentStore.getRootLocation(), contentStoreFilesToDelete); - } - - if (fullContentStoreReplication) - { - deleteUnnecessaryContentStoreFiles(contentStore.getRootLocation()); - } - - contentStore.setLastCommittedVersion(masterContentStoreVersion); - } - - - - final long timeTakenSeconds = getReplicationTimeElapsed(); - final Long bytesDownloadedPerSecond = (timeTakenSeconds != 0 ? new Long(bytesDownloaded/timeTakenSeconds) : null); - LOG.info("Total time taken for download (fullCopy={},bytesDownloaded={}) : {} secs ({} bytes/sec)", - new Object[]{isFullCopyNeeded, bytesDownloaded, timeTakenSeconds, bytesDownloadedPerSecond}); - - if (indexReplicationNeeded) - { - - - Collection> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload); - if (!modifiedConfFiles.isEmpty()) { - reloadCore = true; - downloadConfFiles(confFilesToDownload, latestGeneration); - if (isFullCopyNeeded) { - successfulInstall = solrCore.modifyIndexProps(tmpIdxDirName); - if (successfulInstall) deleteTmpIdxDir = false; - } else { - successfulInstall = moveIndexFiles(tmpIndexDir, indexDir); - } - if (tlogFilesToDownload != null) { - // move tlog files and refresh ulog only if we successfully installed a new index - successfulInstall &= moveTlogFiles(tmpTlogDir); - } - if (successfulInstall) { - if (isFullCopyNeeded) { - // let the system know we are changing dir's and the old one - // may be closed - if (indexDir != null) { - solrCore.getDirectoryFactory().doneWithDirectory(indexDir); - // Cleanup all index files not associated with any *named* snapshot. - solrCore.deleteNonSnapshotIndexFiles(indexDirPath); - } - } - - LOG.info("Configuration files are modified, core will be reloaded"); - logReplicationTimeAndConfFiles(modifiedConfFiles, - successfulInstall);// write to a file time of replication and - // conf files. - } - } else { - terminateAndWaitFsyncService(); - if (isFullCopyNeeded) { - successfulInstall = solrCore.modifyIndexProps(tmpIdxDirName); - if (successfulInstall) deleteTmpIdxDir = false; - } else { - successfulInstall = moveIndexFiles(tmpIndexDir, indexDir); - } - if (tlogFilesToDownload != null) { - // move tlog files and refresh ulog only if we successfully installed a new index - successfulInstall &= moveTlogFiles(tmpTlogDir); - } - if (successfulInstall) { - logReplicationTimeAndConfFiles(modifiedConfFiles, - successfulInstall); - } - } - } - } finally { - if (!isFullCopyNeeded && indexReplicationNeeded) { - solrCore.getUpdateHandler().getSolrCoreState().openIndexWriter(solrCore); - } - } - - - if (indexReplicationNeeded) - { - - // we must reload the core after we open the IW back up - if (successfulInstall && (reloadCore || forceCoreReload)) { - LOG.info("Reloading SolrCore {}", solrCore.getName()); - reloadCore(); - } - - if (successfulInstall) { - if (isFullCopyNeeded) { - // let the system know we are changing dir's and the old one - // may be closed - if (indexDir != null) { - LOG.info("removing old index directory " + indexDir); - solrCore.getDirectoryFactory().doneWithDirectory(indexDir); - solrCore.getDirectoryFactory().remove(indexDir); - } - } - if (isFullCopyNeeded) { - solrCore.getUpdateHandler().newIndexWriter(isFullCopyNeeded); - } - - openNewSearcherAndUpdateCommitPoint(); - } - - if (!isFullCopyNeeded && !forceReplication && !successfulInstall) { - cleanup(solrCore, tmpIndexDir, indexDir, deleteTmpIdxDir, tmpTlogDir, successfulInstall); - cleanupDone = true; - // we try with a full copy of the index - LOG.warn( - "Replication attempt was not successful - trying a full index replication reloadCore={}", - reloadCore); - successfulInstall = fetchLatestIndex(true, reloadCore).getSuccessful(); - } - } - - markReplicationStop(); - return successfulInstall ? IndexFetchResult.INDEX_FETCH_SUCCESS : IndexFetchResult.INDEX_FETCH_FAILURE; - } catch (ReplicationHandlerException e) { - LOG.error("User aborted Replication"); - return new IndexFetchResult(IndexFetchResult.FAILED_BY_EXCEPTION_MESSAGE, false, e); - } catch (SolrException e) { - throw e; - } catch (InterruptedException e) { - throw new InterruptedException("Index fetch interrupted"); - } catch (Exception e) { - throw new SolrException(ErrorCode.SERVER_ERROR, "Index fetch failed : ", e); - } - } finally { - if (!cleanupDone) { - cleanup(solrCore, tmpIndexDir, indexDir, deleteTmpIdxDir, tmpTlogDir, successfulInstall); - } - } - } - - /** - * Download content store required files. - */ - private long downloadContentStoreFiles(String contentStoreDirectory) throws Exception { - LOG.info("Starting download of content store files from master: " + tlogFilesToDownload); - contentStoreFilesDownloaded = Collections.synchronizedList(new ArrayList<>()); - long bytesDownloaded = 0; - - File tmpContentStoreDirectory = new File(contentStoreDirectory, "contentstore." + getDateAsStr(new Date())); - - LOG.info("content store replication: {} files to downlad", contentStoreFilesToDownload.size()); - - List> contentStoreFilesToDownloadFiltered = contentStoreFilesToDownload.stream().filter( - file -> compareContentStoreFiles(new File(contentStoreDirectory), - (String) file.get(NAME), - (Long) file.get(SIZE), - (Long) file.get(CHECKSUM))).collect(Collectors.toList()); - - if (contentStoreFilesToDownloadFiltered.size() != contentStoreFilesToDownload.size()) - { - LOG.warn("content store replication: some of the files are already in sync. {} files to download", contentStoreFilesToDownloadFiltered.size()); - } - - - if (!contentStoreFilesToDownloadFiltered.isEmpty()) { - for (List> partition : Lists.partition(contentStoreFilesToDownloadFiltered, CONTENT_STORE_PARTITION_SIZE)) { - contentStoreFileFetcher = new ContentStoreFetcher( - tmpContentStoreDirectory, - ReplicationHandler.CONTENT_STORE_FILES, - partition); - - contentStoreFileFetcher.fetchContentStore(); - bytesDownloaded += contentStoreFileFetcher.getBytesDownloaded(); - contentStoreFilesDownloaded.addAll(partition); - } - - terminateAndWaitFsyncService(); - copyTmpContentStoreToContentStore(tmpContentStoreDirectory, contentStoreDirectory); - - delTree(tmpContentStoreDirectory); - LOG.info("content store files successfully downloaded"); - } - - - return bytesDownloaded; - } - - - private void cleanup(final SolrCore core, Directory tmpIndexDir, - Directory indexDir, boolean deleteTmpIdxDir, File tmpTlogDir, boolean successfulInstall) throws IOException { - try { - if (!successfulInstall) { - try { - logReplicationTimeAndConfFiles(null, successfulInstall); - } catch (Exception e) { - // this can happen on shutdown, a fetch may be running in a thread after DirectoryFactory is closed - LOG.warn("Could not log failed replication details", e); - } - } - - if (core.getCoreContainer().isZooKeeperAware()) { - // we only track replication success in SolrCloud mode - core.getUpdateHandler().getSolrCoreState().setLastReplicateIndexSuccess(successfulInstall); - } - - filesToDownload = filesDownloaded = confFilesDownloaded = confFilesToDownload = tlogFilesToDownload - = tlogFilesDownloaded = contentStoreFilesDownloaded = contentStoreFilesToDownload = null; - markReplicationStop(); - dirFileFetcher = null; - localFileFetcher = null; - if (fsyncService != null && !fsyncService.isShutdown()) fsyncService.shutdown(); - fsyncService = null; - stop = false; - fsyncException = null; - } finally { - // order below is important - try { - if (tmpIndexDir != null && deleteTmpIdxDir) { - core.getDirectoryFactory().doneWithDirectory(tmpIndexDir); - core.getDirectoryFactory().remove(tmpIndexDir); - } - } catch (Exception e) { - SolrException.log(LOG, e); - } finally { - try { - if (tmpIndexDir != null) core.getDirectoryFactory().release(tmpIndexDir); - } catch (Exception e) { - SolrException.log(LOG, e); - } - try { - if (indexDir != null) { - core.getDirectoryFactory().release(indexDir); - } - } catch (Exception e) { - SolrException.log(LOG, e); - } - try { - if (tmpTlogDir != null) delTree(tmpTlogDir); - } catch (Exception e) { - SolrException.log(LOG, e); - } - } - } - } - - private boolean hasUnusedFiles(Directory indexDir, IndexCommit commit) throws IOException { - String segmentsFileName = commit.getSegmentsFileName(); - SegmentInfos infos = SegmentInfos.readCommit(indexDir, segmentsFileName); - Set currentFiles = new HashSet<>(infos.files(true)); - String[] allFiles = indexDir.listAll(); - for (String file : allFiles) { - if (!file.equals(segmentsFileName) && !currentFiles.contains(file) && !file.endsWith(".lock")) { - LOG.info("Found unused file: " + file); - return true; - } - } - return false; - } - - private volatile Exception fsyncException; - - /** - * terminate the fsync service and wait for all the tasks to complete. If it is already terminated - */ - private void terminateAndWaitFsyncService() throws Exception { - if (fsyncService.isTerminated()) return; - fsyncService.shutdown(); - // give a long wait say 1 hr - fsyncService.awaitTermination(3600, TimeUnit.SECONDS); - // if any fsync failed, throw that exception back - Exception fsyncExceptionCopy = fsyncException; - if (fsyncExceptionCopy != null) throw fsyncExceptionCopy; - } - - /** - * Helper method to record the last replication's details so that we can show them on the statistics page across - * restarts. - * @throws IOException on IO error - */ - @SuppressForbidden(reason = "Need currentTimeMillis for debugging/stats") - private void logReplicationTimeAndConfFiles(Collection> modifiedConfFiles, boolean successfulInstall) throws IOException { - List confFiles = new ArrayList<>(); - if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty()) - for (Map map1 : modifiedConfFiles) - confFiles.add((String) map1.get(NAME)); - - Properties props = replicationHandler.loadReplicationProperties(); - long replicationTime = System.currentTimeMillis(); - long replicationTimeTaken = getReplicationTimeElapsed(); - Directory dir = null; - try { - dir = solrCore.getDirectoryFactory().get(solrCore.getDataDir(), DirContext.META_DATA, solrCore.getSolrConfig().indexConfig.lockType); - - int indexCount = 1, confFilesCount = 1; - if (props.containsKey(TIMES_INDEX_REPLICATED)) { - indexCount = Integer.parseInt(props.getProperty(TIMES_INDEX_REPLICATED)) + 1; - } - StringBuilder sb = readToStringBuilder(replicationTime, props.getProperty(INDEX_REPLICATED_AT_LIST)); - props.setProperty(INDEX_REPLICATED_AT_LIST, sb.toString()); - props.setProperty(INDEX_REPLICATED_AT, String.valueOf(replicationTime)); - props.setProperty(PREVIOUS_CYCLE_TIME_TAKEN, String.valueOf(replicationTimeTaken)); - props.setProperty(TIMES_INDEX_REPLICATED, String.valueOf(indexCount)); - if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty()) { - props.setProperty(CONF_FILES_REPLICATED, confFiles.toString()); - props.setProperty(CONF_FILES_REPLICATED_AT, String.valueOf(replicationTime)); - if (props.containsKey(TIMES_CONFIG_REPLICATED)) { - confFilesCount = Integer.parseInt(props.getProperty(TIMES_CONFIG_REPLICATED)) + 1; - } - props.setProperty(TIMES_CONFIG_REPLICATED, String.valueOf(confFilesCount)); - } - - props.setProperty(LAST_CYCLE_BYTES_DOWNLOADED, String.valueOf(getTotalBytesDownloaded())); - if (!successfulInstall) { - int numFailures = 1; - if (props.containsKey(TIMES_FAILED)) { - numFailures = Integer.parseInt(props.getProperty(TIMES_FAILED)) + 1; - } - props.setProperty(TIMES_FAILED, String.valueOf(numFailures)); - props.setProperty(REPLICATION_FAILED_AT, String.valueOf(replicationTime)); - sb = readToStringBuilder(replicationTime, props.getProperty(REPLICATION_FAILED_AT_LIST)); - props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString()); - } - - - String tmpFileName = REPLICATION_PROPERTIES + "." + System.nanoTime(); - final IndexOutput out = dir.createOutput(tmpFileName, DirectoryFactory.IOCONTEXT_NO_CACHE); - Writer outFile = new OutputStreamWriter(new PropertiesOutputStream(out), StandardCharsets.UTF_8); - try { - props.store(outFile, "Replication details"); - dir.sync(Collections.singleton(tmpFileName)); - } finally { - IOUtils.closeQuietly(outFile); - } - - solrCore.getDirectoryFactory().renameWithOverwrite(dir, tmpFileName, REPLICATION_PROPERTIES); - } catch (Exception e) { - LOG.warn("Exception while updating statistics", e); - } finally { - if (dir != null) { - solrCore.getDirectoryFactory().release(dir); - } - } - } - - long getTotalBytesDownloaded() { - long bytesDownloaded = 0; - //get size from list of files to download - for (Map file : getFilesDownloaded()) { - bytesDownloaded += (Long) file.get(SIZE); - } - - //get size from list of conf files to download - for (Map file : getConfFilesDownloaded()) { - bytesDownloaded += (Long) file.get(SIZE); - } - - //get size from current file being downloaded - Map currentFile = getCurrentFile(); - if (currentFile != null) { - if (currentFile.containsKey("bytesDownloaded")) { - bytesDownloaded += (Long) currentFile.get("bytesDownloaded"); - } - } - return bytesDownloaded; - } - - private StringBuilder readToStringBuilder(long replicationTime, String str) { - StringBuilder sb = new StringBuilder(); - List l = new ArrayList<>(); - if (str != null && str.length() != 0) { - String[] ss = str.split(","); - Collections.addAll(l, ss); - } - sb.append(replicationTime); - if (!l.isEmpty()) { - for (int i = 0; i < l.size() || i < 9; i++) { - if (i == l.size() || i == 9) break; - String s = l.get(i); - sb.append(",").append(s); - } - } - return sb; - } - - private void openNewSearcherAndUpdateCommitPoint() throws IOException { - RefCounted searcher = null; - IndexCommit commitPoint; - // must get the latest solrCore object because the one we have might be closed because of a reload - // todo stop keeping solrCore around - SolrCore core = solrCore.getCoreContainer().getCore(solrCore.getName()); - try { - Future[] waitSearcher = new Future[1]; - searcher = core.getSearcher(true, true, waitSearcher, true); - if (waitSearcher[0] != null) { - try { - waitSearcher[0].get(); - } catch (InterruptedException | ExecutionException e) { - SolrException.log(LOG, e); - } - } - commitPoint = searcher.get().getIndexReader().getIndexCommit(); - } finally { - if (searcher != null) { - searcher.decref(); - } - core.close(); - } - - // update the commit point in replication handler - replicationHandler.indexCommitPoint = commitPoint; - - } - - private void reloadCore() { - final CountDownLatch latch = new CountDownLatch(1); - new Thread(() -> { - try { - solrCore.getCoreContainer().reload(solrCore.getName()); - } catch (Exception e) { - LOG.error("Could not reload core ", e); - } finally { - latch.countDown(); - } - }).start(); - try { - latch.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted while waiting for core reload to finish", e); - } - } - - private void downloadConfFiles(List> confFilesToDownload, long latestGeneration) throws Exception { - LOG.info("Starting download of configuration files from master: " + confFilesToDownload); - confFilesDownloaded = Collections.synchronizedList(new ArrayList<>()); - File tmpconfDir = new File(solrCore.getResourceLoader().getConfigDir(), "conf." + getDateAsStr(new Date())); - try { - boolean status = tmpconfDir.mkdirs(); - if (!status) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Failed to create temporary config folder: " + tmpconfDir.getName()); - } - for (Map file : confFilesToDownload) { - String saveAs = (String) (file.get(ALIAS) == null ? file.get(NAME) : file.get(ALIAS)); - localFileFetcher = new LocalFsFileFetcher(tmpconfDir, file, saveAs, CONF_FILE_SHORT, latestGeneration); - currentFile = file; - localFileFetcher.fetchFile(); - confFilesDownloaded.add(new HashMap<>(file)); - } - // this is called before copying the files to the original conf dir - // so that if there is an exception avoid corrupting the original files. - terminateAndWaitFsyncService(); - copyTmpConfFiles2Conf(tmpconfDir); - } finally { - delTree(tmpconfDir); - } - } - - /** - * Download all the tlog files to the temp tlog directory. - */ - private long downloadTlogFiles(File tmpTlogDir, long latestGeneration) throws Exception { - LOG.info("Starting download of tlog files from master: " + tlogFilesToDownload); - tlogFilesDownloaded = Collections.synchronizedList(new ArrayList<>()); - long bytesDownloaded = 0; - - boolean status = tmpTlogDir.mkdirs(); - if (!status) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Failed to create temporary tlog folder: " + tmpTlogDir.getName()); - } - for (Map file : tlogFilesToDownload) { - String saveAs = (String) (file.get(ALIAS) == null ? file.get(NAME) : file.get(ALIAS)); - localFileFetcher = new LocalFsFileFetcher(tmpTlogDir, file, saveAs, TLOG_FILE, latestGeneration); - currentFile = file; - localFileFetcher.fetchFile(); - bytesDownloaded += localFileFetcher.getBytesDownloaded(); - tlogFilesDownloaded.add(new HashMap<>(file)); - } - return bytesDownloaded; - } - - - - /** - * Download the index files. If a new index is needed, download all the files. - * - * @param downloadCompleteIndex is it a fresh index copy - * @param tmpIndexDir the directory to which files need to be downloadeed to - * @param indexDir the indexDir to be merged to - * @param latestGeneration the version number - * - * @return number of bytes downloaded - */ - private long downloadIndexFiles(boolean downloadCompleteIndex, Directory indexDir, Directory tmpIndexDir, long latestGeneration) - throws Exception { - if (LOG.isDebugEnabled()) { - LOG.debug("Download files to dir: " + Arrays.asList(indexDir.listAll())); - } - long bytesDownloaded = 0; - for (Map file : filesToDownload) { - String filename = (String) file.get(NAME); - long size = (Long) file.get(SIZE); - CompareResult compareResult = compareFile(indexDir, filename, size, (Long) file.get(CHECKSUM)); - boolean alwaysDownload = filesToAlwaysDownloadIfNoChecksums(filename, size, compareResult); - - LOG.debug("Downloading file={} size={} checksum={} alwaysDownload={}", filename, size, file.get(CHECKSUM), alwaysDownload); - - if (!compareResult.equal || downloadCompleteIndex || alwaysDownload) { - dirFileFetcher = new DirectoryFileFetcher(tmpIndexDir, file, - (String) file.get(NAME), FILE, latestGeneration); - currentFile = file; - dirFileFetcher.fetchFile(); - bytesDownloaded += dirFileFetcher.getBytesDownloaded(); - filesDownloaded.add(new HashMap<>(file)); - } else { - LOG.info("Skipping download for " + file.get(NAME) - + " because it already exists"); - } - } - return bytesDownloaded; - } - - private static boolean filesToAlwaysDownloadIfNoChecksums(String filename, - long size, CompareResult compareResult) { - // without checksums to compare, we always download .si, .liv, segments_N, - // and any very small files - return !compareResult.checkSummed && (filename.endsWith(".si") || filename.endsWith(".liv") - || filename.startsWith("segments_") || size < _100K); - } - - protected static class CompareResult { - boolean equal = false; - boolean checkSummed = false; - } - - - /** - * - * Check if the same version of the file to download is already in the contentstore. - * - * @param contentStoreDirectoryRoot the content store top directory. - * @param filename the filename. - * @param length the file length. - * @param checksum the file chechsum. - * @return true if the same file already exists in the contentstore. - */ - private static boolean compareContentStoreFiles(File contentStoreDirectoryRoot, String filename, long length, long checksum){ - File f = new File(contentStoreDirectoryRoot.getAbsolutePath() + "/" + filename); - if (f.length() != length) { - return true; - } - - Checksum localCheckSum = new Adler32(); - - if (getCheckSum(localCheckSum, f) != checksum) - { - return true; - } - - LOG.debug("{} already in contentstore", filename); - return false; - } - - private static CompareResult compareFile(Directory indexDir, String filename, Long backupIndexFileLen, Long backupIndexFileChecksum) { - CompareResult compareResult = new CompareResult(); - try { - try (final IndexInput indexInput = indexDir.openInput(filename, IOContext.READONCE)) { - long indexFileLen = indexInput.length(); - long indexFileChecksum = 0; - - if (backupIndexFileChecksum != null) { - try { - indexFileChecksum = CodecUtil.retrieveChecksum(indexInput); - compareResult.checkSummed = true; - } catch (Exception e) { - LOG.warn("Could not retrieve checksum from file.", e); - } - } - - if (!compareResult.checkSummed) { - // we don't have checksums to compare - - if (indexFileLen == backupIndexFileLen) { - compareResult.equal = true; - return compareResult; - } else { - LOG.info( - "File {} did not match. expected length is {} and actual length is {}", filename, backupIndexFileLen, indexFileLen); - compareResult.equal = false; - return compareResult; - } - } - - // we have checksums to compare - - if (indexFileLen == backupIndexFileLen && indexFileChecksum == backupIndexFileChecksum) { - compareResult.equal = true; - return compareResult; - } else { - LOG.warn("File {} did not match. expected checksum is {} and actual is checksum {}. " + - "expected length is {} and actual length is {}", filename, backupIndexFileChecksum, indexFileChecksum, - backupIndexFileLen, indexFileLen); - compareResult.equal = false; - return compareResult; - } - } - } catch (NoSuchFileException | FileNotFoundException e) { - compareResult.equal = false; - return compareResult; - } catch (IOException e) { - LOG.error("Could not read file " + filename + ". Downloading it again", e); - compareResult.equal = false; - return compareResult; - } - } - - /** Returns true if the file exists (can be opened), false - * if it cannot be opened, and (unlike Java's - * File.exists) throws IOException if there's some - * unexpected error. */ - private static boolean slowFileExists(Directory dir, String fileName) throws IOException { - try { - dir.openInput(fileName, IOContext.DEFAULT).close(); - return true; - } catch (NoSuchFileException | FileNotFoundException e) { - return false; - } - } - - /** - * All the files which are common between master and slave must have same size and same checksum else we assume - * they are not compatible (stale). - * - * @return true if the index stale and we need to download a fresh copy, false otherwise. - * @throws IOException if low level io error - */ - private boolean isIndexStale(Directory dir) throws IOException { - for (Map file : filesToDownload) { - String filename = (String) file.get(NAME); - Long length = (Long) file.get(SIZE); - Long checksum = (Long) file.get(CHECKSUM); - if (slowFileExists(dir, filename)) { - if (checksum != null) { - if (!(compareFile(dir, filename, length, checksum).equal)) { - // file exists and size or checksum is different, therefore we must download it again - return true; - } - } else { - if (length != dir.fileLength(filename)) { - LOG.warn("File {} did not match. expected length is {} and actual length is {}", filename, length, dir.fileLength(filename)); - return true; - } - } - } - } - return false; - } - - /** - * Copy a file by the File#renameTo() method. If it fails, it is considered a failure - *

- */ - private boolean moveAFile(Directory tmpIdxDir, Directory indexDir, String fname) { - LOG.debug("Moving file: {}", fname); - boolean success = false; - try { - if (slowFileExists(indexDir, fname)) { - LOG.warn("Cannot complete replication attempt because file already exists:" + fname); - - // we fail - we downloaded the files we need, if we can't move one in, we can't - // count on the correct index - return false; - } - } catch (IOException e) { - SolrException.log(LOG, "could not check if a file exists", e); - return false; - } - try { - solrCore.getDirectoryFactory().move(tmpIdxDir, indexDir, fname, DirectoryFactory.IOCONTEXT_NO_CACHE); - success = true; - } catch (IOException e) { - SolrException.log(LOG, "Could not move file", e); - } - return success; - } - - /** - * Copy all index files from the temp index dir to the actual index. The segments_N file is copied last. - */ - private boolean moveIndexFiles(Directory tmpIdxDir, Directory indexDir) { - if (LOG.isDebugEnabled()) { - try { - LOG.info("From dir files:" + Arrays.asList(tmpIdxDir.listAll())); - LOG.info("To dir files:" + Arrays.asList(indexDir.listAll())); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - String segmentsFile = null; - for (Map f : filesDownloaded) { - String fname = (String) f.get(NAME); - // the segments file must be copied last - // or else if there is a failure in between the - // index will be corrupted - if (fname.startsWith("segments_")) { - //The segments file must be copied in the end - //Otherwise , if the copy fails index ends up corrupted - segmentsFile = fname; - continue; - } - if (!moveAFile(tmpIdxDir, indexDir, fname)) return false; - } - //copy the segments file last - if (segmentsFile != null) { - return moveAFile(tmpIdxDir, indexDir, segmentsFile); - } - return true; - } - - /** - *

- * Copy all the tlog files from the temp tlog dir to the actual tlog dir, and reset - * the {@link UpdateLog}. The copy will try to preserve the original tlog directory - * if the copy fails. - *

- *

- * This assumes that the tlog files transferred from the leader are in synch with the - * index files transferred from the leader. The reset of the update log relies on the version - * of the latest operations found in the tlog files. If the tlogs are ahead of the latest commit - * point, it will not copy all the needed buffered updates for the replay and it will miss - * some operations. - *

- */ - private boolean moveTlogFiles(File tmpTlogDir) { - UpdateLog ulog = solrCore.getUpdateHandler().getUpdateLog(); - - VersionInfo vinfo = ulog.getVersionInfo(); - vinfo.blockUpdates(); // block updates until the new update log is initialised - try { - // reset the update log before copying the new tlog directory - CdcrUpdateLog.BufferedUpdates bufferedUpdates = ((CdcrUpdateLog) ulog).resetForRecovery(); - // try to move the temp tlog files to the tlog directory - if (!copyTmpTlogFiles2Tlog(tmpTlogDir)) return false; - // reinitialise the update log and copy the buffered updates - if (bufferedUpdates.tlog != null) { - // map file path to its new backup location - File parentDir = FileSystems.getDefault().getPath(solrCore.getUpdateHandler().getUpdateLog().getLogDir()).getParent().toFile(); - File backupTlogDir = new File(parentDir, tmpTlogDir.getName()); - bufferedUpdates.tlog = new File(backupTlogDir, bufferedUpdates.tlog.getName()); - } - // init the update log with the new set of tlog files, and copy the buffered updates - ((CdcrUpdateLog) ulog).initForRecovery(bufferedUpdates.tlog, bufferedUpdates.offset); - } - catch (Exception e) { - LOG.error("Unable to copy tlog files", e); - return false; - } - finally { - vinfo.unblockUpdates(); - } - return true; - } - - /** - * Make file list - */ - private List makeTmpConfDirFileList(File dir, List fileList) { - File[] files = dir.listFiles(); - for (File file : files) { - if (file.isFile()) { - fileList.add(file); - } else if (file.isDirectory()) { - fileList = makeTmpConfDirFileList(file, fileList); - } - } - return fileList; - } - - /** - * The conf files are copied to the tmp dir to the conf dir. A backup of the old file is maintained - */ - private void copyTmpConfFiles2Conf(File tmpconfDir) { - boolean status = false; - File confDir = new File(solrCore.getResourceLoader().getConfigDir()); - for (File file : makeTmpConfDirFileList(tmpconfDir, new ArrayList<>())) { - File oldFile = new File(confDir, file.getPath().substring(tmpconfDir.getPath().length(), file.getPath().length())); - if (!oldFile.getParentFile().exists()) { - status = oldFile.getParentFile().mkdirs(); - if (!status) { - throw new SolrException(ErrorCode.SERVER_ERROR, - "Unable to mkdirs: " + oldFile.getParentFile()); - } - } - if (oldFile.exists()) { - File backupFile = new File(oldFile.getPath() + "." + getDateAsStr(new Date(oldFile.lastModified()))); - if (!backupFile.getParentFile().exists()) { - status = backupFile.getParentFile().mkdirs(); - if (!status) { - throw new SolrException(ErrorCode.SERVER_ERROR, - "Unable to mkdirs: " + backupFile.getParentFile()); - } - } - status = oldFile.renameTo(backupFile); - if (!status) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Unable to rename: " + oldFile + " to: " + backupFile); - } - } - status = file.renameTo(oldFile); - if (!status) { - throw new SolrException(ErrorCode.SERVER_ERROR, - "Unable to rename: " + file + " to: " + oldFile); - } - } - } - - - - private void copyTmpContentStoreToContentStore(File tmpContentStoreDir, String contentStorePath) throws IOException { - - String tmpContentStorePath = tmpContentStoreDir.getPath(); - - try - { - Files.walk(tmpContentStoreDir.toPath()).forEach(p -> { - File tmpFile = new File(p.toUri()); - if (!tmpFile.isDirectory()) { - File csFile = new File(p.toString().replaceFirst(tmpContentStorePath, contentStorePath)); - try { - Files.createDirectories(Paths.get(csFile.getParent())); - tmpFile.renameTo(csFile); - } catch (IOException e) { - LOG.error("impossible to copy {}", csFile.toString()); - } - } - }); - } catch (IOException e) { - LOG.error("impossible tmp content store"); - throw e; - } - } - - - private void deleteContentStoreFiles(String contentStorePath, List> filesToDelete) - { - filesToDelete.stream().map(f -> (String) f.get(NAME)).forEach( p -> - { - File f = new File(contentStorePath, p); - f.delete(); - } - ); - - LOG.info("deleted {} files from content store", filesToDelete.size()); - } - - private void deleteUnnecessaryContentStoreFiles(String contentStorePath) - { - - AtomicInteger fileDeleted = new AtomicInteger(); - Set fileNames = contentStoreFilesToDownload.stream().map(e -> (String) e.get(NAME)).collect(Collectors.toSet()); - try - { - Files.walk(Paths.get(contentStorePath)).forEach(p -> { - File f = new File(p.toUri()); - if (!f.isDirectory() && !fileNames.contains(p.toString().replaceFirst(contentStorePath, ""))) { - try { - Files.delete(p); - fileDeleted.getAndIncrement(); - } catch (IOException ex) { - LOG.error("Impossible delete file {}", p); - } - } - }); - } catch (IOException e) { - LOG.error("Impossible to delete unnecessary files. Content store may contains unused contents"); - } - - LOG.info("deleted {} unnecessary files from content store", fileDeleted); - } - - /** - * The tlog files are moved from the tmp dir to the tlog dir as an atomic filesystem operation. - * A backup of the old directory is maintained. If the directory move fails, it will try to revert back the original - * tlog directory. - */ - private boolean copyTmpTlogFiles2Tlog(File tmpTlogDir) { - Path tlogDir = FileSystems.getDefault().getPath(solrCore.getUpdateHandler().getUpdateLog().getLogDir()); - Path backupTlogDir = FileSystems.getDefault().getPath(tlogDir.getParent().toAbsolutePath().toString(), tmpTlogDir.getName()); - - try { - Files.move(tlogDir, backupTlogDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { - SolrException.log(LOG, "Unable to rename: " + tlogDir + " to: " + backupTlogDir, e); - return false; - } - - Path src = FileSystems.getDefault().getPath(backupTlogDir.toAbsolutePath().toString(), tmpTlogDir.getName()); - try { - Files.move(src, tlogDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { - SolrException.log(LOG, "Unable to rename: " + src + " to: " + tlogDir, e); - - // In case of error, try to revert back the original tlog directory - try { - Files.move(backupTlogDir, tlogDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e2) { - // bad, we were not able to revert back the original tlog directory - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Unable to rename: " + backupTlogDir + " to: " + tlogDir); - } - - return false; - } - - return true; - } - - - private String getDateAsStr(Date d) { - return new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(d); - } - - private final Map confFileInfoCache = new HashMap<>(); - - /** - * The local conf files are compared with the conf files in the master. If they are same (by checksum) do not copy. - * - * @param confFilesToDownload The list of files obtained from master - * - * @return a list of configuration files which have changed on the master and need to be downloaded. - */ - @SuppressWarnings("unchecked") - private Collection> getModifiedConfFiles(List> confFilesToDownload) { - if (confFilesToDownload == null || confFilesToDownload.isEmpty()) - return Collections.emptyList(); - //build a map with alias/name as the key - Map> nameVsFile = new HashMap<>(); - NamedList names = new NamedList(); - for (Map map : confFilesToDownload) { - //if alias is present that is the name the file may have in the slave - String name = (String) (map.get(ALIAS) == null ? map.get(NAME) : map.get(ALIAS)); - nameVsFile.put(name, map); - names.add(name, null); - } - //get the details of the local conf files with the same alias/name - List> localFilesInfo = replicationHandler.getConfFileInfoFromCache(names, confFileInfoCache); - //compare their size/checksum to see if - for (Map fileInfo : localFilesInfo) { - String name = (String) fileInfo.get(NAME); - Map m = nameVsFile.get(name); - if (m == null) continue; // the file is not even present locally (so must be downloaded) - if (m.get(CHECKSUM).equals(fileInfo.get(CHECKSUM))) { - nameVsFile.remove(name); //checksums are same so the file need not be downloaded - } - } - return nameVsFile.isEmpty() ? Collections.EMPTY_LIST : nameVsFile.values(); - } - - /** - * This simulates File.delete exception-wise, since this class has some strange behavior with it. - * The only difference is it returns null on success, throws SecurityException on SecurityException, - * otherwise returns Throwable preventing deletion (instead of false), for additional information. - */ - static Throwable delete(File file) { - try { - Files.delete(file.toPath()); - return null; - } catch (SecurityException e) { - throw e; - } catch (Throwable other) { - return other; - } - } - - static boolean delTree(File dir) { - try { - org.apache.lucene.util.IOUtils.rm(dir.toPath()); - return true; - } catch (IOException e) { - LOG.warn("Unable to delete directory : " + dir, e); - return false; - } - } - - /** - * Stops the ongoing fetch - */ - void abortFetch() { - stop = true; - } - - @SuppressForbidden(reason = "Need currentTimeMillis for debugging/stats") - private void markReplicationStart() { - replicationTimer = new RTimer(); - replicationStartTimeStamp = new Date(); - } - - private void markReplicationStop() { - replicationStartTimeStamp = null; - replicationTimer = null; - } - - Date getReplicationStartTimeStamp() { - return replicationStartTimeStamp; - } - - long getReplicationTimeElapsed() { - long timeElapsed = 0; - if (replicationStartTimeStamp != null) - timeElapsed = TimeUnit.SECONDS.convert((long) replicationTimer.getTime(), TimeUnit.MILLISECONDS); - return timeElapsed; - } - - List> getTlogFilesToDownload() { - //make a copy first because it can be null later - List> tmp = tlogFilesToDownload; - //create a new instance. or else iterator may fail - return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); - } - - List> getTlogFilesDownloaded() { - //make a copy first because it can be null later - List> tmp = tlogFilesDownloaded; - // NOTE: it's safe to make a copy of a SynchronizedCollection(ArrayList) - return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); - } - - List> getConfFilesToDownload() { - //make a copy first because it can be null later - List> tmp = confFilesToDownload; - //create a new instance. or else iterator may fail - return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); - } - - List> getConfFilesDownloaded() { - //make a copy first because it can be null later - List> tmp = confFilesDownloaded; - // NOTE: it's safe to make a copy of a SynchronizedCollection(ArrayList) - return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); - } - - List> getContentStoreFilesDownloaded() { - //make a copy first because it can be null later - List> tmp = contentStoreFilesDownloaded; - // NOTE: it's safe to make a copy of a SynchronizedCollection(ArrayList) - return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); - } - - List> getFilesToDownload() { - //make a copy first because it can be null later - List> tmp = filesToDownload; - return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); - } - - List> getContentStoreFileToDownload() { - //make a copy first because it can be null later - List> tmp = contentStoreFilesToDownload; - return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); - } - - - List> getFilesDownloaded() { - List> tmp = filesDownloaded; - return tmp == null ? Collections.emptyList() : new ArrayList<>(tmp); - } - - // TODO: currently does not reflect conf files - Map getCurrentFile() { - Map tmp = currentFile; - DirectoryFileFetcher tmpFileFetcher = dirFileFetcher; - if (tmp == null) - return null; - tmp = new HashMap<>(tmp); - if (tmpFileFetcher != null) - tmp.put("bytesDownloaded", tmpFileFetcher.getBytesDownloaded()); - return tmp; - } - - private static class ReplicationHandlerException extends InterruptedException { - ReplicationHandlerException(String message) { - super(message); - } - } - - private interface FileInterface { - void sync() throws IOException; - void write(byte[] buf, int packetSize) throws IOException; - void close() throws Exception; - void delete() throws Exception; - } - - /** - * The class acts as a client for ReplicationHandler.FileStream. It understands the protocol of wt=filestream - */ - public class FileFetcher { - protected FileInterface file; - String fileName; - String saveAs; - boolean includeChecksum = true; - final String solrParamOutput; - final Long indexGen; - - protected long size; - long bytesDownloaded = 0; - protected byte[] buf = new byte[1024 * 1024]; - final Checksum checksum; - int errorCount = 0; - boolean aborted = false; - - FileFetcher(FileInterface file, Map fileDetails, String saveAs, - String solrParamOutput, long latestGen) { - this.file = file; - this.fileName = (String) fileDetails.get(NAME); - this.size = (Long) fileDetails.get(SIZE); - this.solrParamOutput = solrParamOutput; - this.saveAs = saveAs; - indexGen = latestGen; - - if (includeChecksum) { - checksum = new Adler32(); - } else { - checksum = null; - } - } - - - FileFetcher( String solrParamOutput, long latestGen) { - - this.solrParamOutput = solrParamOutput; - indexGen = latestGen; - if (includeChecksum) { - checksum = new Adler32(); - } else { - checksum = null; - } - } - - long getBytesDownloaded() { - return bytesDownloaded; - } - - /** - * The main method which downloads file - */ - void fetchFile() throws Exception { - bytesDownloaded = 0; - try { - fetch(); - } catch(Exception e) { - if (!aborted) { - SolrException.log(IndexFetcher.LOG, "Error fetching file, doing one retry...", e); - // one retry - fetch(); - } else { - throw e; - } - } - } - - protected void fetch() throws Exception { - try { - while (true) { - final FastInputStream is = getStream(); - int result; - try { - //fetch packets one by one in a single request - result = fetchPackets(is); - if (result == 0 || result == NO_CONTENT) { - return; - } - //if there is an error continue. But continue from the point where it got broken - } finally { - IOUtils.closeQuietly(is); - } - } - } finally { - cleanup(); - //if cleanup succeeds . The file is downloaded fully. do an fsync - fsyncService.submit(() -> { - try { - file.sync(); - } catch (IOException e) { - fsyncException = e; - } - }); - } - } - - protected int fetchPackets(FastInputStream fis) throws Exception { - byte[] intbytes = new byte[4]; - byte[] longbytes = new byte[8]; - try { - - while (true) { - if (stop) { - stop = false; - aborted = true; - throw new ReplicationHandlerException("User aborted replication"); - } - long checkSumServer = -1; - fis.readFully(intbytes); - //read the size of the packet - int packetSize = readInt(intbytes); - if (packetSize <= 0) { - LOG.warn("No content received for file: {}", fileName); - return NO_CONTENT; - } - if (buf.length < packetSize) - buf = new byte[packetSize]; - if (checksum != null) { - //read the checksum - fis.readFully(longbytes); - checkSumServer = readLong(longbytes); - } - //then read the packet of bytes - fis.readFully(buf, 0, packetSize); - //compare the checksum as sent from the master - if (includeChecksum) { - checksum.reset(); - checksum.update(buf, 0, packetSize); - long checkSumClient = checksum.getValue(); - if (checkSumClient != checkSumServer) { - LOG.error("Checksum not matched between client and server for file: {}", fileName); - //if checksum is wrong it is a problem return for retry - return 1; - } - } - //if everything is fine, write down the packet to the file - file.write(buf, packetSize); - bytesDownloaded += packetSize; - LOG.debug("Fetched and wrote {} bytes of file: {}", bytesDownloaded, fileName); - if (bytesDownloaded >= size) - return 0; - //errorCount is always set to zero after a successful packet - errorCount = 0; - } - } catch (ReplicationHandlerException e) { - throw e; - } catch (Exception e) { - LOG.warn("Error in fetching file: {} (downloaded {} of {} bytes)", fileName, bytesDownloaded, size, e); - //for any failure, increment the error count - errorCount++; - //if it fails for the same packet for MAX_RETRIES fail and come out - if (errorCount > MAX_RETRIES) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Failed to fetch file: " + fileName + - " (downloaded " + bytesDownloaded + " of " + size + " bytes" + - ", error count: " + errorCount + " > " + MAX_RETRIES + ")", e); - } - return ERR; - } - } - - /** - * The webcontainer flushes the data only after it fills the buffer size. So, all data has to be read as readFully() - * other wise it fails. So read everything as bytes and then extract an integer out of it - */ - int readInt(byte[] b) { - return (((b[0] & 0xff) << 24) | ((b[1] & 0xff) << 16) - | ((b[2] & 0xff) << 8) | (b[3] & 0xff)); - - } - - /** - * Same as above but to read longs from a byte array - */ - long readLong(byte[] b) { - return (((long) (b[0] & 0xff)) << 56) | (((long) (b[1] & 0xff)) << 48) - | (((long) (b[2] & 0xff)) << 40) | (((long) (b[3] & 0xff)) << 32) - | (((long) (b[4] & 0xff)) << 24) | ((b[5] & 0xff) << 16) - | ((b[6] & 0xff) << 8) | ((b[7] & 0xff)); - - } - - /** - * cleanup everything - */ - private void cleanup() { - try { - file.close(); - } catch (Exception e) {/* no-op */ - LOG.error("Error closing file: {}", this.saveAs, e); - } - if (bytesDownloaded != size) { - //if the download is not complete then - //delete the file being downloaded - try { - file.delete(); - } catch (Exception e) { - LOG.error("Error deleting file: {}", this.saveAs, e); - } - //if the failure is due to a user abort it is returned normally else an exception is thrown - if (!aborted) - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Unable to download " + fileName + " completely. Downloaded " - + bytesDownloaded + "!=" + size); - } - } - - /** - * Open a new stream using HttpClient - */ - protected FastInputStream getStream() throws IOException { - - ModifiableSolrParams params = new ModifiableSolrParams(); - -// //the method is command=filecontent - params.set(COMMAND, CMD_GET_FILE); - params.set(GENERATION, Long.toString(indexGen)); - params.set(CommonParams.QT, ReplicationHandler.PATH); - //add the version to download. This is used to reserve the download - params.set(solrParamOutput, fileName); - if (useInternalCompression) { - params.set(COMPRESSION, "true"); - } -// use checksum - if (this.includeChecksum) { - params.set(CHECKSUM, true); - } - //wt=filestream this is a custom protocol - params.set(CommonParams.WT, FILE_STREAM); - // This happen if there is a failure there is a retry. the offset= ensures that - // the server starts from the offset - if (bytesDownloaded > 0) { - params.set(OFFSET, Long.toString(bytesDownloaded)); - } - - - NamedList response; - InputStream is = null; - - // TODO use shardhandler - try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl) - .withHttpClient(myHttpClient) - .withResponseParser(null) - .build() - ) { - client.setSoTimeout(60000); - client.setConnectionTimeout(15000); - QueryRequest req = new QueryRequest(params); - response = client.request(req); - is = (InputStream) response.get("stream"); - if(useInternalCompression) { - is = new InflaterInputStream(is); - } - return new FastInputStream(is); - } catch (Exception e) { - //close stream on error - org.apache.commons.io.IOUtils.closeQuietly(is); - throw new IOException("Could not download file '" + fileName + "'", e); - } - } - } - - private static class DirectoryFile implements FileInterface { - private final String saveAs; - private Directory copy2Dir; - private IndexOutput outStream; - - DirectoryFile(Directory tmpIndexDir, String saveAs) throws IOException { - this.saveAs = saveAs; - this.copy2Dir = tmpIndexDir; - outStream = copy2Dir.createOutput(this.saveAs, DirectoryFactory.IOCONTEXT_NO_CACHE); - } - - public void sync() throws IOException { - copy2Dir.sync(Collections.singleton(saveAs)); - } - - public void write(byte[] buf, int packetSize) throws IOException { - outStream.writeBytes(buf, 0, packetSize); - } - - public void close() throws Exception { - outStream.close(); - } - - public void delete() throws Exception { - copy2Dir.deleteFile(saveAs); - } - } - - private class DirectoryFileFetcher extends FileFetcher { - DirectoryFileFetcher(Directory tmpIndexDir, Map fileDetails, String saveAs, - String solrParamOutput, long latestGen) throws IOException { - super(new DirectoryFile(tmpIndexDir, saveAs), fileDetails, saveAs, solrParamOutput, latestGen); - } - } - - private static class LocalFsFile implements FileInterface { - private File copy2Dir; - - FileChannel fileChannel; - private FileOutputStream fileOutputStream; - File file; - - LocalFsFile(File dir, String saveAs) throws IOException { - this.copy2Dir = dir; - - this.file = new File(copy2Dir, saveAs); - - File parentDir = this.file.getParentFile(); - if( ! parentDir.exists() ){ - if ( ! parentDir.mkdirs() ) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Failed to create (sub)directory for file: " + saveAs); - } - } - - this.fileOutputStream = new FileOutputStream(file); - this.fileChannel = this.fileOutputStream.getChannel(); - } - - public void sync() throws IOException { - FileUtils.sync(file); - } - - public void write(byte[] buf, int packetSize) throws IOException { - fileChannel.write(ByteBuffer.wrap(buf, 0, packetSize)); - } - - public void close() throws Exception { - //close the FileOutputStream (which also closes the Channel) - fileOutputStream.close(); - } - - public void delete() throws Exception { - Files.delete(file.toPath()); - } - } - - class LocalFsFileFetcher extends FileFetcher { - LocalFsFileFetcher(File dir, Map fileDetails, String saveAs, - String solrParamOutput, long latestGen) throws IOException { - super(new LocalFsFile(dir, saveAs), fileDetails, saveAs, solrParamOutput, latestGen); - } - } - - - class ContentStoreFetcher extends FileFetcher { - private final File dir; - private final Set filesToDownload; - private final Set filesDownloaded; - - ContentStoreFetcher(File dir, - String solrParamOutput, - List> filesDetails) throws IOException { - super(solrParamOutput, 0); - this.dir = dir; - this.filesToDownload = new HashSet<>(); - this.filesDownloaded = new HashSet<>(); - filesDetails.forEach( e -> filesToDownload.add((String) e.get(NAME))); - } - - @Override - protected void fetch() throws Exception { - - while (true) { - final FastInputStream is = getStream(); - int result; - try { - //fetch packets one by one in a single request - result = fetchPackets(is); - if (result == 0 || result == NO_CONTENT) { - return; - } - //if there is an error continue. But continue from the point where it got broken - } finally { - IOUtils.closeQuietly(is); - } - } - } - - - public void fetchContentStore() throws Exception { - this.fetchFile(); - } - - - @Override - protected FastInputStream getStream() throws IOException { - - ModifiableSolrParams params = new ModifiableSolrParams(); - - params.set(COMMAND, CMD_CONTENT_STORE_FILES); - params.set(GENERATION, Long.toString(indexGen)); - params.set(CommonParams.QT, ReplicationHandler.PATH); - - List l = new ArrayList<>(); - params.set(CONTENT_STORE_FILE_LIST, - filesToDownload.toArray(String[]::new)); - - //add the version to download. This is used to reserve the download -// params.set(solrParamOutput, fileName); - if (useInternalCompression) { - params.set(COMPRESSION, "true"); - } - //use checksum - if (this.includeChecksum) { - params.set(CHECKSUM, true); - } - //wt=filestream this is a custom protocol - params.set(CommonParams.WT, FILE_STREAM); - // This happen if there is a failure there is a retry. the offset= ensures that - // the server starts from the offset - if (bytesDownloaded > 0) { - params.set(OFFSET, Long.toString(bytesDownloaded)); - } - - - NamedList response; - InputStream is = null; - - // TODO use shardhandler - try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl) - .withHttpClient(myHttpClient) - .withResponseParser(null) - .build() - ) { - client.setSoTimeout(60000); - client.setConnectionTimeout(15000); - QueryRequest req = new QueryRequest(params); - response = client.request(req); - is = (InputStream) response.get("stream"); - if(useInternalCompression) { - is = new InflaterInputStream(is); - } - return new FastInputStream(is); - } catch (Exception e) { - //close stream on error - org.apache.commons.io.IOUtils.closeQuietly(is); - throw new IOException("Could not download file '" + fileName + "'", e); - } - } - - - @Override - protected int fetchPackets(FastInputStream fis) throws Exception { - byte[] intbytes = new byte[4]; - byte[] longbytes = new byte[8]; - try { - while (true) { - - int fileNameSize; - - try { - fis.readFully(intbytes); - fileNameSize = readInt(intbytes); - } catch (EOFException e) { - LOG.debug("Fetched the whole batch of files"); - return 0; - } - - byte[] filenameBytes = new byte[fileNameSize]; - fis.readFully(filenameBytes, 0, fileNameSize); - - String fileName = new String(filenameBytes); - - // Error, file not requested - if (!filesToDownload.contains(fileName)){ - throw new Exception("file " + fileName + " not requested"); - } - - FileInterface file = new LocalFsFile(dir, fileName); - fis.readFully(intbytes); - - int fileSize = readInt(intbytes); - - - long fileSizeDownloaded = 0; - if (fileSize == 0) { - file.close(); - return 0; - } - - while (true) { - if (stop) { - stop = false; - aborted = true; - throw new ReplicationHandlerException("User aborted replication"); - } - long checkSumServer = -1; - fis.readFully(intbytes); - //read the size of the packet - int packetSize = readInt(intbytes); - if (packetSize <= 0) { - LOG.warn("No content received"); - file.close(); - return NO_CONTENT; - } - - if (buf.length < packetSize) { - buf = new byte[packetSize]; - } - - if (checksum != null) { - //read the checksum - fis.readFully(longbytes); - checkSumServer = readLong(longbytes); - } - - //then read the packet of bytes - fis.readFully(buf, 0, packetSize); - //compare the checksum as sent from the master - if (includeChecksum) { - checksum.reset(); - checksum.update(buf, 0, packetSize); - long checkSumClient = checksum.getValue(); - if (checkSumClient != checkSumServer) { - LOG.warn("Checksum not matched between client and server for file: {}", this.fileName); - //if checksum is wrong it is a problem return for retry - file.close(); - return ERR; - } - } - //if everything is fine, write down the packet to the file - file.write(buf, packetSize); - fileSizeDownloaded += packetSize; - - if (fileSizeDownloaded >= fileSize) { - file.close(); - bytesDownloaded += fileSizeDownloaded; - - //Todo check if it is necessary - fsyncService.submit(() -> { - try { - file.sync(); - } catch (IOException e) { - fsyncException = e; - } - }); - filesToDownload.remove(fileName); - filesDownloaded.add(fileName); - break; - } - //errorCount is always set to zero after a successful packet - errorCount = 0; - } - - LOG.debug("downloaded content store file: {}", fileName); - } - } catch (ReplicationHandlerException e) { - throw e; - } catch (Exception e) { - LOG.warn("Error in fetching file: {} (downloaded {} of {} bytes)", - new Object[]{ fileName, bytesDownloaded, size, e}); - //for any failure, increment the error count - errorCount++; - //if it fails for the same packet for MAX_RETRIES fail and come out - if (errorCount > MAX_RETRIES) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Failed to fetch content store bucket: " + - " (downloaded " + filesDownloaded.size() + " files out of " + filesToDownload.size() + filesDownloaded.size() + - ", error count: " + errorCount + " > " + MAX_RETRIES + ")", e); - } - return ERR; - } - } - - public Set getFilesDownloaded() { - return filesDownloaded; - } - } - - NamedList getDetails() throws IOException, SolrServerException { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(COMMAND, CMD_DETAILS); - params.set("slave", false); - params.set(CommonParams.QT, ReplicationHandler.PATH); - - // TODO use shardhandler - try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) { - client.setSoTimeout(60000); - client.setConnectionTimeout(15000); - QueryRequest request = new QueryRequest(params); - return client.request(request); - } - } - - public void destroy() { - abortFetch(); - } - - String getMasterUrl() { - return masterUrl; - } - - private static final int MAX_RETRIES = 5; - - private static final int NO_CONTENT = 1; - - private static final int ERR = 2; - - public static final String REPLICATION_PROPERTIES = "replication.properties"; - - static final String INDEX_REPLICATED_AT = "indexReplicatedAt"; - - static final String TIMES_INDEX_REPLICATED = "timesIndexReplicated"; - - static final String CONF_FILES_REPLICATED = "confFilesReplicated"; - - static final String CONF_FILES_REPLICATED_AT = "confFilesReplicatedAt"; - - static final String TIMES_CONFIG_REPLICATED = "timesConfigReplicated"; - - static final String LAST_CYCLE_BYTES_DOWNLOADED = "lastCycleBytesDownloaded"; - - static final String TIMES_FAILED = "timesFailed"; - - static final String REPLICATION_FAILED_AT = "replicationFailedAt"; - - static final String PREVIOUS_CYCLE_TIME_TAKEN = "previousCycleTimeInSeconds"; - - static final String INDEX_REPLICATED_AT_LIST = "indexReplicatedAtList"; - - static final String REPLICATION_FAILED_AT_LIST = "replicationFailedAtList"; -} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java deleted file mode 100644 index 612749613..000000000 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/ReplicationHandler.java +++ /dev/null @@ -1,1955 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.alfresco.solr.handler; - -import org.alfresco.solr.AlfrescoCoreAdminHandler; -import org.alfresco.solr.content.SolrContentStore; -import org.apache.commons.io.IOUtils; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.IndexDeletionPolicy; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.SegmentCommitInfo; -import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.RateLimiter; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.CoreAdminParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.FastOutputStream; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.SuppressForbidden; -import org.apache.solr.core.CloseHook; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.DirectoryFactory.DirContext; -import org.apache.solr.core.IndexDeletionPolicyWrapper; -import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrDeletionPolicy; -import org.apache.solr.core.SolrEventListener; -import org.apache.solr.core.backup.repository.BackupRepository; -import org.apache.solr.core.backup.repository.LocalFileSystemRepository; -import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; -import org.apache.solr.handler.RequestHandlerBase; -import org.apache.solr.handler.RestoreCore; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.update.CdcrUpdateLog; -import org.apache.solr.update.SolrIndexWriter; -import org.apache.solr.update.VersionInfo; -import org.apache.solr.util.DefaultSolrThreadFactory; -import org.apache.solr.util.NumberUtils; -import org.apache.solr.util.PropertiesInputStream; -import org.apache.solr.util.RefCounted; -import org.apache.solr.util.plugin.SolrCoreAware; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.lang.invoke.MethodHandles; -import java.net.URI; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.charset.StandardCharsets; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.zip.Adler32; -import java.util.zip.Checksum; -import java.util.zip.DeflaterOutputStream; - -import static org.apache.solr.common.params.CommonParams.NAME; - -/** - *

A Handler which provides a REST API for replication and serves replication requests from Slaves.

- *

When running on the master, it provides the following commands

  1. Get the current replicable index version - * (command=indexversion)
  2. Get the list of files for a given index version - * (command=filelist&indexversion=<VERSION>)
  3. Get full or a part (chunk) of a given index or a config - * file (command=filecontent&file=<FILE_NAME>) You can optionally specify an offset and length to get that - * chunk of the file. You can request a configuration file by using "cf" parameter instead of the "file" parameter.
    4. - *
  4. Get status/statistics (command=details)

When running on the slave, it provides the following - * commands

  1. Perform an index fetch now (command=snappull)
  2. Get status/statistics (command=details)
    3. - *
  3. Abort an index fetch (command=abort)
  4. Enable/Disable polling the master for new versions (command=enablepoll - * or command=disablepoll)
- * - * - * @since solr 1.4 - */ -public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAware { - - public static final String PATH = "/replication"; - - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private SolrCore core; - - private SolrContentStore contentStore; - private boolean contentStoreReplication = false; - - public static final class CommitVersionInfo { - public final long version; - final long generation; - - private CommitVersionInfo(long g, long v) { - generation = g; - version = v; - } - - @Override - public String toString() { - return "Commit [version = " + version + ", generation = " + generation + "]"; - } - - /** - * builds a CommitVersionInfo data for the specified IndexCommit. - * Will never be null, ut version and generation may be zero if - * there are problems extracting them from the commit data - */ - public static CommitVersionInfo build(IndexCommit commit) { - long generation = commit.getGeneration(); - long version = 0; - try { - final Map commitData = commit.getUserData(); - String commitTime = commitData.get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY); - if (commitTime != null) { - try { - version = Long.parseLong(commitTime); - } catch (NumberFormatException e) { - LOG.warn("Version in commitData was not formatted correctly: " + commitTime, e); - } - } - } catch (IOException e) { - LOG.warn("Unable to get version from commitData, commit: " + commit, e); - } - return new CommitVersionInfo(generation, version); - } - } - - private IndexFetcher pollingIndexFetcher; - - private ReentrantLock indexFetchLock = new ReentrantLock(); - - private static Lock contentStoreReplicationLock = new ReentrantLock(); - - private static boolean isContentStoreReplicating = false; - - private ExecutorService restoreExecutor = ExecutorUtil.newMDCAwareSingleThreadExecutor( - new DefaultSolrThreadFactory("restoreExecutor")); - - private volatile Future restoreFuture; - - private volatile String currentRestoreName; - - private String includeConfFiles; - - private NamedList confFileNameAlias = new NamedList<>(); - - private boolean isMaster; - - private boolean isSlave; - - private boolean replicateOnOptimize; - - private boolean replicateOnCommit; - - private boolean replicateOnStart; - - private ScheduledExecutorService executorService; - - private volatile long executorStartTime; - - private int numberBackupsToKeep = 0; //zero: do not delete old backups - - private int numTimesReplicated = 0; - - private final Map confFileInfoCache = new HashMap<>(); - - private Integer reserveCommitDuration = readIntervalMs("00:00:10"); - - volatile IndexCommit indexCommitPoint; - - volatile NamedList snapShootDetails; - - private AtomicBoolean replicationEnabled = new AtomicBoolean(true); - - private Long pollIntervalNs; - private String pollIntervalStr; - - /** - * Disable the timer task for polling - */ - private AtomicBoolean pollDisabled = new AtomicBoolean(false); - - private String getPollInterval() { - return pollIntervalStr; - } - - @Override - public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - rsp.setHttpCaching(false); - final SolrParams solrParams = req.getParams(); - String command = solrParams.get(COMMAND); - if (command == null) { - rsp.add(STATUS, OK_STATUS); - rsp.add("message", "No command"); - return; - } - // This command does not give the current index version of the master - // It gives the current 'replicateable' index version - if (command.equals(CMD_INDEX_VERSION)) { - IndexCommit commitPoint = indexCommitPoint; // make a copy so it won't change - - if (commitPoint == null) { - // if this handler is 'lazy', we may not have tracked the last commit - // because our commit listener is registered on inform - commitPoint = core.getDeletionPolicy().getLatestCommit(); - } - - if (commitPoint != null && replicationEnabled.get()) { - // - // There is a race condition here. The commit point may be changed / deleted by the time - // we get around to reserving it. This is a very small window though, and should not result - // in a catastrophic failure, but will result in the client getting an empty file list for - // the CMD_GET_FILE_LIST command. - // - core.getDeletionPolicy().setReserveDuration(commitPoint.getGeneration(), reserveCommitDuration); - rsp.add(CMD_INDEX_VERSION, IndexDeletionPolicyWrapper.getCommitTimestamp(commitPoint)); - rsp.add(GENERATION, commitPoint.getGeneration()); - rsp.add(CONTENT_STORE_VERSION, contentStore.getLastCommittedVersion()); - } else { - // This happens when replication is not configured to happen after startup and no commit/optimize - // has happened yet. - rsp.add(CMD_INDEX_VERSION, 0L); - rsp.add(GENERATION, 0L); - } - } else if (command.equals(CMD_GET_FILE)) { - getFileStream(solrParams, rsp); - } else if (command.equals(CMD_CONTENT_STORE_FILES)) { - getContetntStore(new ModifiableSolrParams(solrParams), rsp); - }else if (command.equals(CMD_GET_FILE_LIST)) { - getFileList(solrParams, rsp); - } else if (command.equalsIgnoreCase(CMD_BACKUP)) { - doSnapShoot(new ModifiableSolrParams(solrParams), rsp, req); - rsp.add(STATUS, OK_STATUS); - } else if (command.equalsIgnoreCase(CMD_RESTORE)) { - restore(new ModifiableSolrParams(solrParams), rsp, req); - rsp.add(STATUS, OK_STATUS); - } else if (command.equalsIgnoreCase(CMD_RESTORE_STATUS)) { - rsp.add(CMD_RESTORE_STATUS, getRestoreStatus()); - } else if (command.equalsIgnoreCase(CMD_DELETE_BACKUP)) { - deleteSnapshot(new ModifiableSolrParams(solrParams)); - rsp.add(STATUS, OK_STATUS); - } else if (command.equalsIgnoreCase(CMD_FETCH_INDEX)) { - String masterUrl = solrParams.get(MASTER_URL); - if (!isSlave && masterUrl == null) { - rsp.add(STATUS,ERR_STATUS); - rsp.add("message","No slave configured or no 'masterUrl' Specified"); - return; - } - final SolrParams paramsCopy = new ModifiableSolrParams(solrParams); - Thread fetchThread = new Thread(() -> doFetch(paramsCopy, false), "explicit-fetchindex-cmd") ; - fetchThread.setDaemon(false); - fetchThread.start(); - if (solrParams.getBool(WAIT, false)) { - fetchThread.join(); - } - rsp.add(STATUS, OK_STATUS); - } else if (command.equalsIgnoreCase(CMD_DISABLE_POLL)) { - if (pollingIndexFetcher != null){ - disablePoll(); - rsp.add(STATUS, OK_STATUS); - } else { - rsp.add(STATUS, ERR_STATUS); - rsp.add("message","No slave configured"); - } - } else if (command.equalsIgnoreCase(CMD_ENABLE_POLL)) { - if (pollingIndexFetcher != null){ - enablePoll(); - rsp.add(STATUS, OK_STATUS); - }else { - rsp.add(STATUS,ERR_STATUS); - rsp.add("message","No slave configured"); - } - } else if (command.equalsIgnoreCase(CMD_ABORT_FETCH)) { - if (abortFetch()){ - rsp.add(STATUS, OK_STATUS); - } else { - rsp.add(STATUS,ERR_STATUS); - rsp.add("message","No slave configured"); - } - } else if (command.equals(CMD_SHOW_COMMITS)) { - rsp.add(CMD_SHOW_COMMITS, getCommits()); - } else if (command.equals(CMD_DETAILS)) { - rsp.add(CMD_DETAILS, getReplicationDetails(solrParams.getBool("slave", true))); - } else if (CMD_ENABLE_REPL.equalsIgnoreCase(command)) { - replicationEnabled.set(true); - rsp.add(STATUS, OK_STATUS); - } else if (CMD_DISABLE_REPL.equalsIgnoreCase(command)) { - replicationEnabled.set(false); - rsp.add(STATUS, OK_STATUS); - } - } - - private boolean abortFetch() { - IndexFetcher fetcher = currentIndexFetcher; - if (fetcher != null){ - fetcher.abortFetch(); - return true; - } else { - return false; - } - } - - private void deleteSnapshot(ModifiableSolrParams params) { - String name = params.get(NAME); - if(name == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Missing mandatory param: name"); - } - - SnapShooter snapShooter = new SnapShooter(core, params.get(CoreAdminParams.BACKUP_LOCATION), params.get(NAME)); - snapShooter.validateDeleteSnapshot(); - snapShooter.deleteSnapAsync(this); - } - - private List> getCommits() { - Map commits = core.getDeletionPolicy().getCommits(); - List> l = new ArrayList<>(); - - for (IndexCommit c : commits.values()) { - try { - NamedList nl = new NamedList<>(); - nl.add("indexVersion", IndexDeletionPolicyWrapper.getCommitTimestamp(c)); - nl.add(GENERATION, c.getGeneration()); - List commitList = new ArrayList<>(c.getFileNames().size()); - commitList.addAll(c.getFileNames()); - Collections.sort(commitList); - nl.add(CMD_GET_FILE_LIST, commitList); - l.add(nl); - } catch (IOException e) { - LOG.warn("Exception while reading files for commit " + c, e); - } - } - return l; - } - - static long getCheckSum(Checksum checksum, File f) { - checksum.reset(); - byte[] buffer = new byte[1024 * 1024]; - int bytesRead; - try (FileInputStream fis = new FileInputStream(f)) { - while ((bytesRead = fis.read(buffer)) >= 0) { - checksum.update(buffer, 0, bytesRead); - } - return checksum.getValue(); - } catch (Exception e) { - LOG.warn("Exception in finding checksum of " + f, e); - throw new RuntimeException(e); - } - } - - private volatile IndexFetcher currentIndexFetcher; - - boolean acquireContentStoreReplicationTask() { - contentStoreReplicationLock.lock(); - if (!isContentStoreReplicating) - { - contentStoreReplication = true; - isContentStoreReplicating = true; - } - - contentStoreReplicationLock.unlock(); - return contentStoreReplication; - } - - private void releaseContentStoreReplicationTask() - { - if (contentStoreReplication) - { - contentStoreReplicationLock.lock(); - isContentStoreReplicating = false; - contentStoreReplication = false; - contentStoreReplicationLock.unlock(); - } - } - - private IndexFetcher.IndexFetchResult doFetch(SolrParams solrParams, boolean forceReplication) { - String masterUrl = solrParams == null ? null : solrParams.get(MASTER_URL); - if (!indexFetchLock.tryLock()) - return IndexFetcher.IndexFetchResult.LOCK_OBTAIN_FAILED; - try { - - if (masterUrl != null) { - if (currentIndexFetcher != null && currentIndexFetcher != pollingIndexFetcher) { - currentIndexFetcher.destroy(); - } - currentIndexFetcher = new IndexFetcher(solrParams.toNamedList(), this, core, contentStore); - } else { - currentIndexFetcher = pollingIndexFetcher; - } - return currentIndexFetcher.fetchLatestIndex(forceReplication, acquireContentStoreReplicationTask()); - } catch (Exception e) { - SolrException.log(LOG, "Index fetch failed ", e); - return new IndexFetcher.IndexFetchResult(IndexFetcher.IndexFetchResult.FAILED_BY_EXCEPTION_MESSAGE, false, e); - } finally { - if (pollingIndexFetcher != null) { - currentIndexFetcher = pollingIndexFetcher; - } - - releaseContentStoreReplicationTask(); - indexFetchLock.unlock(); - } - } - - private boolean isReplicating() { - return indexFetchLock.isLocked(); - } - - private void restore(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) throws IOException { - if (restoreFuture != null && !restoreFuture.isDone()) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Restore in progress. Cannot run multiple restore operations" + - "for the same core"); - } - String name = params.get(NAME); - String location = params.get(CoreAdminParams.BACKUP_LOCATION); - - String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY); - CoreContainer cc = core.getCoreContainer(); - BackupRepository repo; - if (repoName != null) { - repo = cc.newBackupRepository(Optional.of(repoName)); - location = repo.getBackupLocation(location); - if (location == null) { - throw new IllegalArgumentException("location is required"); - } - } else { - repo = new LocalFileSystemRepository(); - } - - //If location is not provided then assume that the restore index is present inside the data directory. - if (location == null) { - location = core.getDataDir(); - } - - URI locationUri = repo.createURI(location); - - //If name is not provided then look for the last unnamed( the ones with the snapshot.timestamp format) - //snapshot folder since we allow snapshots to be taken without providing a name. Pick the latest timestamp. - if (name == null) { - String[] filePaths = repo.listAll(locationUri); - List dirs = new ArrayList<>(); - for (String f : filePaths) { - OldBackupDirectory obd = new OldBackupDirectory(locationUri, f); - if (obd.getTimestamp().isPresent()) { - dirs.add(obd); - } - } - Collections.sort(dirs); - if (dirs.size() == 0) { - throw new SolrException(ErrorCode.BAD_REQUEST, "No backup name specified and none found in " + core.getDataDir()); - } - name = dirs.get(0).getDirName(); - } else { - //"snapshot." is prefixed by snapshooter - name = "snapshot." + name; - } - - RestoreCore restoreCore = new RestoreCore(repo, core, locationUri, name); - try { - MDC.put("RestoreCore.core", core.getName()); - MDC.put("RestoreCore.backupLocation", location); - MDC.put("RestoreCore.backupName", name); - restoreFuture = restoreExecutor.submit(restoreCore); - currentRestoreName = name; - } finally { - MDC.remove("RestoreCore.core"); - MDC.remove("RestoreCore.backupLocation"); - MDC.remove("RestoreCore.backupName"); - } - } - - private NamedList getRestoreStatus() { - NamedList status = new SimpleOrderedMap<>(); - - if (restoreFuture == null) { - status.add(STATUS, "No restore actions in progress"); - return status; - } - - status.add("snapshotName", currentRestoreName); - if (restoreFuture.isDone()) { - try { - boolean success = restoreFuture.get(); - if (success) { - status.add(STATUS, SUCCESS); - } else { - status.add(STATUS, FAILED); - } - } catch (Exception e) { - status.add(STATUS, FAILED); - status.add(EXCEPTION, e.getMessage()); - } - } else { - status.add(STATUS, "In Progress"); - } - return status; - } - - private void doSnapShoot(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) { - try { - int numberToKeep = params.getInt(NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM, 0); - if (numberToKeep > 0 && numberBackupsToKeep > 0) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Cannot use " - + NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM + " if " - + NUMBER_BACKUPS_TO_KEEP_INIT_PARAM - + " was specified in the configuration."); - } - numberToKeep = Math.max(numberToKeep, numberBackupsToKeep); - if (numberToKeep < 1) { - numberToKeep = Integer.MAX_VALUE; - } - - IndexCommit indexCommit; - String commitName = params.get(CoreAdminParams.COMMIT_NAME); - if (commitName != null) { - SolrSnapshotMetaDataManager snapshotMgr = core.getSnapshotMetaDataManager(); - Optional commit = snapshotMgr.getIndexCommitByName(commitName); - if(commit.isPresent()) { - indexCommit = commit.get(); - } else { - throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an index commit with name " + commitName + - " for core " + core.getName()); - } - } else { - IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy(); - indexCommit = delPolicy.getLatestCommit(); - - if (indexCommit == null) { - indexCommit = req.getSearcher().getIndexReader().getIndexCommit(); - } - } - - String location = params.get(CoreAdminParams.BACKUP_LOCATION); - String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY); - CoreContainer cc = core.getCoreContainer(); - BackupRepository repo; - if (repoName != null) { - repo = cc.newBackupRepository(Optional.of(repoName)); - location = repo.getBackupLocation(location); - if (location == null) { - throw new IllegalArgumentException("location is required"); - } - } else { - repo = new LocalFileSystemRepository(); - if (location == null) { - location = core.getDataDir(); - } else { - location = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString(); - } - } - - // small race here before the commit point is saved - URI locationUri = repo.createURI(location); - SnapShooter snapShooter = new SnapShooter(repo, core, locationUri, params.get(NAME), commitName); - snapShooter.validateCreateSnapshot(); - snapShooter.createSnapAsync(indexCommit, numberToKeep, (nl) -> snapShootDetails = nl); - - } catch (Exception e) { - LOG.warn("Exception during creating a snapshot", e); - rsp.add("exception", e); - } - } - - /** - * This method adds an Object of FileStream to the response . The FileStream implements a custom protocol which is - * understood by IndexFetcher.FileFetcher - */ - private void getFileStream(SolrParams solrParams, SolrQueryResponse rsp) { - ModifiableSolrParams rawParams = new ModifiableSolrParams(solrParams); - rawParams.set(CommonParams.WT, FILE_STREAM); - - String cfileName = solrParams.get(CONF_FILE_SHORT); - String tlogFileName = solrParams.get(TLOG_FILE); - if (cfileName != null) { - rsp.add(FILE_STREAM, new LocalFsConfFileStream(solrParams)); - } else if (tlogFileName != null ) { - rsp.add(FILE_STREAM, new LocalFsTlogFileStream(solrParams)); - } else { - rsp.add(FILE_STREAM, new DirectoryFileStream(solrParams)); - } - } - - - private void getContetntStore(SolrParams solrParams, SolrQueryResponse rsp){ - rsp.add(FILE_STREAM, new ContentStoreFilesStream(solrParams)); - } - - private void getFileList(SolrParams solrParams, SolrQueryResponse rsp) { - String v = solrParams.get(GENERATION); - - if (v == null) { - rsp.add("status", "no index generation specified"); - return; - } - - long indexGeneration = Long.parseLong(v); - - v = solrParams.get(CONTENT_STORE_VERSION); - if (v == null){ - rsp.add("status", "no content store generation specified"); - return; - } - - long contentStoreGeneration = Long.parseLong(v); - - - if (indexGeneration != NO_INDEX_REPLICATION_REQUIRED){ - - - IndexCommit commit = core.getDeletionPolicy().getCommitPoint(indexGeneration); - - if (commit == null) { - rsp.add("status", "invalid index generation"); - return; - } - - // reserve the indexcommit for sometime - core.getDeletionPolicy().setReserveDuration(indexGeneration, reserveCommitDuration); - List> result = new ArrayList<>(); - Directory dir = null; - try { - dir = core.getDirectoryFactory().get(core.getNewIndexDir(), DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); - SegmentInfos infos = SegmentInfos.readCommit(dir, commit.getSegmentsFileName()); - - for (SegmentCommitInfo commitInfo : infos) { - for (String file : commitInfo.files()) { - Map fileMeta = new HashMap<>(); - fileMeta.put(NAME, file); - fileMeta.put(SIZE, dir.fileLength(file)); - - try (final IndexInput in = dir.openInput(file, IOContext.READONCE)) { - try { - long checksum = CodecUtil.retrieveChecksum(in); - fileMeta.put(CHECKSUM, checksum); - } catch (Exception e) { - LOG.warn("Could not read checksum from index file: " + file, e); - } - } - - result.add(fileMeta); - } - } - - // add the segments_N file - - Map fileMeta = new HashMap<>(); - fileMeta.put(NAME, infos.getSegmentsFileName()); - fileMeta.put(SIZE, dir.fileLength(infos.getSegmentsFileName())); - if (infos.getId() != null) { - try (final IndexInput in = dir.openInput(infos.getSegmentsFileName(), IOContext.READONCE)) { - try { - fileMeta.put(CHECKSUM, CodecUtil.retrieveChecksum(in)); - } catch (Exception e) { - LOG.warn("Could not read checksum from index file: " + infos.getSegmentsFileName(), e); - } - } - } - result.add(fileMeta); - } catch (IOException e) { - rsp.add("status", "unable to get file names for given index generation"); - rsp.add(EXCEPTION, e); - LOG.error("Unable to get file names for indexCommit generation: " + indexGeneration, e); - } finally { - if (dir != null) { - try { - core.getDirectoryFactory().release(dir); - } catch (IOException e) { - SolrException.log(LOG, "Could not release directory after fetching file list", e); - } - } - } - rsp.add(CMD_GET_FILE_LIST, result); - - // fetch list of tlog files only if cdcr is activated - if (solrParams.getBool(TLOG_FILES, true) && core.getUpdateHandler().getUpdateLog() != null - && core.getUpdateHandler().getUpdateLog() instanceof CdcrUpdateLog) { - try { - List> tlogfiles = getTlogFileList(commit); - LOG.info("Adding tlog files to list: " + tlogfiles); - rsp.add(TLOG_FILES, tlogfiles); - } - catch (IOException e) { - rsp.add("status", "unable to get tlog file names for given index generation"); - rsp.add(EXCEPTION, e); - LOG.error("Unable to get tlog file names for indexCommit generation: " + indexGeneration, e); - } - } - - if (confFileNameAlias.size() < 1 || core.getCoreContainer().isZooKeeperAware()) { - return; - } - LOG.debug("Adding config files to list: " + includeConfFiles); - //if configuration files need to be included get their details - rsp.add(CONF_FILES, getConfFileInfoFromCache(confFileNameAlias, confFileInfoCache)); - } - - if (contentStoreGeneration != SolrContentStore.NO_CONTENT_STORE_REPLICATION_REQUIRED) - { - Map>> changes = contentStore.getChanges(contentStoreGeneration); - rsp.add(CONTENT_STORE_FILES, changes); - } - } - - /** - * Retrieves the list of tlog files associated to a commit point. - */ - private List> getTlogFileList(IndexCommit commit) throws IOException { - long maxVersion = this.getMaxVersion(commit); - CdcrUpdateLog ulog = (CdcrUpdateLog) core.getUpdateHandler().getUpdateLog(); - String[] logList = ulog.getLogList(new File(ulog.getLogDir())); - List> tlogFiles = new ArrayList<>(); - for (String fileName : logList) { - // filter out tlogs that are older than the current index commit generation, so that the list of tlog files is - // in synch with the latest index commit point - long startVersion = Math.abs(Long.parseLong(fileName.substring(fileName.lastIndexOf('.') + 1))); - if (startVersion < maxVersion) { - Map fileMeta = new HashMap<>(); - fileMeta.put(NAME, fileName); - fileMeta.put(SIZE, new File(ulog.getLogDir(), fileName).length()); - tlogFiles.add(fileMeta); - } - } - return tlogFiles; - } - - /** - * Retrieves the maximum version number from an index commit. - */ - private long getMaxVersion(IndexCommit commit) throws IOException { - try (DirectoryReader reader = DirectoryReader.open(commit)) { - IndexSearcher searcher = new IndexSearcher(reader); - VersionInfo vinfo = core.getUpdateHandler().getUpdateLog().getVersionInfo(); - return Math.abs(vinfo.getMaxVersionFromIndex(searcher)); - } - } - - /** - * For configuration files, checksum of the file is included because, unlike index files, they may have same content - * but different timestamps. - *

- * The local conf files information is cached so that everytime it does not have to compute the checksum. The cache is - * refreshed only if the lastModified of the file changes - */ - List> getConfFileInfoFromCache(NamedList nameAndAlias, - final Map confFileInfoCache) { - List> confFiles = new ArrayList<>(); - synchronized (confFileInfoCache) { - File confDir = new File(core.getResourceLoader().getConfigDir()); - Checksum checksum = null; - for (int i = 0; i < nameAndAlias.size(); i++) { - String cf = nameAndAlias.getName(i); - File f = new File(confDir, cf); - if (!f.exists() || f.isDirectory()) continue; //must not happen - FileInfo info = confFileInfoCache.get(cf); - if (info == null || info.lastmodified != f.lastModified() || info.size != f.length()) { - if (checksum == null) checksum = new Adler32(); - info = new FileInfo(f.lastModified(), cf, f.length(), getCheckSum(checksum, f)); - confFileInfoCache.put(cf, info); - } - Map m = info.getAsMap(); - if (nameAndAlias.getVal(i) != null) m.put(ALIAS, nameAndAlias.getVal(i)); - confFiles.add(m); - } - } - return confFiles; - } - - - - public static class FileInfo { - long lastmodified; - String name; - long size; - long checksum; - - public FileInfo(File file, String name) - { - Checksum checksum = new Adler32(); - - this.lastmodified = file.lastModified(); - this.name = name; - this.size = file.length(); - this.checksum = getCheckSum(checksum, file); - } - - FileInfo(long lasmodified, String name, long size, long checksum) { - this.lastmodified = lasmodified; - this.name = name; - this.size = size; - this.checksum = checksum; - } - - public Map getAsMap() { - Map map = new HashMap<>(); - map.put(NAME, name); - map.put(SIZE, size); - map.put(CHECKSUM, checksum); - return map; - } - } - - private void disablePoll() { - if (isSlave) { - pollDisabled.set(true); - LOG.info("inside disable poll, value of pollDisabled = " + pollDisabled); - } - } - - private void enablePoll() { - if (isSlave) { - pollDisabled.set(false); - LOG.info("inside enable poll, value of pollDisabled = " + pollDisabled); - } - } - - private boolean isPollingDisabled() { - return pollDisabled.get(); - } - - @SuppressForbidden(reason = "Need currentTimeMillis, to output next execution time in replication details") - private void markScheduledExecutionStart() { - executorStartTime = System.currentTimeMillis(); - } - - private Date getNextScheduledExecTime() { - Date nextTime = null; - if (executorStartTime > 0) - nextTime = new Date(executorStartTime + TimeUnit.MILLISECONDS.convert(pollIntervalNs, TimeUnit.NANOSECONDS)); - return nextTime; - } - - @SuppressWarnings("unused") - int getTimesReplicatedSinceStartup() { - return numTimesReplicated; - } - - @SuppressWarnings("unused") - void setTimesReplicatedSinceStartup() { - numTimesReplicated++; - } - - @Override - public Category getCategory() { - return Category.REPLICATION; - } - - @Override - public String getDescription() { - return "ReplicationHandler provides replication of index and configuration files from Master to Slaves"; - } - - /** - * returns the CommitVersionInfo for the current searcher, or null on error. - */ - private CommitVersionInfo getIndexVersion() { - CommitVersionInfo v = null; - RefCounted searcher = core.getSearcher(); - try { - v = CommitVersionInfo.build(searcher.get().getIndexReader().getIndexCommit()); - } catch (IOException e) { - LOG.warn("Unable to get index commit: ", e); - } finally { - searcher.decref(); - } - return v; - } - - @Override - @SuppressWarnings("unchecked") - public NamedList getStatistics() { - NamedList list = super.getStatistics(); - if (core != null) { - list.add("indexSize", NumberUtils.readableSize(core.getIndexSize())); - CommitVersionInfo vInfo = (core != null && !core.isClosed()) ? getIndexVersion(): null; - list.add("indexVersion", null == vInfo ? 0 : vInfo.version); - list.add(GENERATION, null == vInfo ? 0 : vInfo.generation); - - list.add("indexPath", core.getIndexDir()); - list.add("isMaster", String.valueOf(isMaster)); - list.add("isSlave", String.valueOf(isSlave)); - - IndexFetcher fetcher = currentIndexFetcher; - if (fetcher != null) { - list.add(MASTER_URL, fetcher.getMasterUrl()); - if (getPollInterval() != null) { - list.add(POLL_INTERVAL, getPollInterval()); - } - list.add("isPollingDisabled", String.valueOf(isPollingDisabled())); - list.add("isReplicating", String.valueOf(isReplicating())); - long elapsed = fetcher.getReplicationTimeElapsed(); - long val = fetcher.getTotalBytesDownloaded(); - if (elapsed > 0) { - list.add("timeElapsed", elapsed); - list.add("bytesDownloaded", val); - list.add("downloadSpeed", val / elapsed); - } - Properties props = loadReplicationProperties(); - addVal(list, IndexFetcher.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class); - addVal(list, IndexFetcher.INDEX_REPLICATED_AT, props, Date.class); - addVal(list, IndexFetcher.CONF_FILES_REPLICATED_AT, props, Date.class); - addVal(list, IndexFetcher.REPLICATION_FAILED_AT, props, Date.class); - addVal(list, IndexFetcher.TIMES_FAILED, props, Integer.class); - addVal(list, IndexFetcher.TIMES_INDEX_REPLICATED, props, Integer.class); - addVal(list, IndexFetcher.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class); - addVal(list, IndexFetcher.TIMES_CONFIG_REPLICATED, props, Integer.class); - addVal(list, IndexFetcher.CONF_FILES_REPLICATED, props, String.class); - } - if (isMaster) { - if (includeConfFiles != null) list.add("confFilesToReplicate", includeConfFiles); - list.add(REPLICATE_AFTER, getReplicateAfterStrings()); - list.add("replicationEnabled", String.valueOf(replicationEnabled.get())); - } - } - return list; - } - - /** - * Used for showing statistics and progress information. - */ - private NamedList getReplicationDetails(boolean showSlaveDetails) { - NamedList details = new SimpleOrderedMap<>(); - NamedList master = new SimpleOrderedMap<>(); - NamedList slave = new SimpleOrderedMap<>(); - - details.add("indexSize", NumberUtils.readableSize(core.getIndexSize())); - details.add("indexPath", core.getIndexDir()); - details.add(CMD_SHOW_COMMITS, getCommits()); - details.add("isMaster", String.valueOf(isMaster)); - details.add("isSlave", String.valueOf(isSlave)); - CommitVersionInfo vInfo = getIndexVersion(); - details.add("indexVersion", null == vInfo ? 0 : vInfo.version); - details.add("robavaria", 50); - details.add(GENERATION, null == vInfo ? 0 : vInfo.generation); - - IndexCommit commit = indexCommitPoint; // make a copy so it won't change - - if (isMaster) { - if (includeConfFiles != null) master.add(CONF_FILES, includeConfFiles); - master.add(REPLICATE_AFTER, getReplicateAfterStrings()); - master.add("replicationEnabled", String.valueOf(replicationEnabled.get())); - } - - if (isMaster && commit != null) { - CommitVersionInfo repCommitInfo = CommitVersionInfo.build(commit); - master.add("replicableVersion", repCommitInfo.version); - master.add("replicableGeneration", repCommitInfo.generation); - } - - IndexFetcher fetcher = currentIndexFetcher; - if (fetcher != null) { - Properties props = loadReplicationProperties(); - if (showSlaveDetails) { - try { - NamedList nl = fetcher.getDetails(); - slave.add("masterDetails", nl.get(CMD_DETAILS)); - } catch (Exception e) { - LOG.warn( - "Exception while invoking 'details' method for replication on master ", - e); - slave.add(ERR_STATUS, "invalid_master"); - } - } - slave.add(MASTER_URL, fetcher.getMasterUrl()); - if (getPollInterval() != null) { - slave.add(POLL_INTERVAL, getPollInterval()); - } - Date nextScheduled = getNextScheduledExecTime(); - if (nextScheduled != null && !isPollingDisabled()) { - slave.add(NEXT_EXECUTION_AT, nextScheduled.toString()); - } else if (isPollingDisabled()) { - slave.add(NEXT_EXECUTION_AT, "Polling disabled"); - } - addVal(slave, IndexFetcher.INDEX_REPLICATED_AT, props, Date.class); - addVal(slave, IndexFetcher.INDEX_REPLICATED_AT_LIST, props, List.class); - addVal(slave, IndexFetcher.REPLICATION_FAILED_AT_LIST, props, List.class); - addVal(slave, IndexFetcher.TIMES_INDEX_REPLICATED, props, Integer.class); - addVal(slave, IndexFetcher.CONF_FILES_REPLICATED, props, Integer.class); - addVal(slave, IndexFetcher.TIMES_CONFIG_REPLICATED, props, Integer.class); - addVal(slave, IndexFetcher.CONF_FILES_REPLICATED_AT, props, Integer.class); - addVal(slave, IndexFetcher.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class); - addVal(slave, IndexFetcher.TIMES_FAILED, props, Integer.class); - addVal(slave, IndexFetcher.REPLICATION_FAILED_AT, props, Date.class); - addVal(slave, IndexFetcher.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class); - - slave.add("currentDate", new Date().toString()); - slave.add("isPollingDisabled", String.valueOf(isPollingDisabled())); - boolean isReplicating = isReplicating(); - slave.add("isReplicating", String.valueOf(isReplicating)); - if (isReplicating) { - try { - long bytesToDownload = 0; - List filesToDownload = new ArrayList<>(); - for (Map file : fetcher.getFilesToDownload()) { - filesToDownload.add((String) file.get(NAME)); - bytesToDownload += (Long) file.get(SIZE); - } - - //get list of conf files to download - for (Map file : fetcher.getConfFilesToDownload()) { - filesToDownload.add((String) file.get(NAME)); - bytesToDownload += (Long) file.get(SIZE); - } - - //get list of conf files to download - for (Map file : fetcher.getContentStoreFileToDownload()) { - filesToDownload.add((String) file.get(NAME)); - bytesToDownload += (Long) file.get(SIZE); - } - - slave.add("filesToDownload", filesToDownload); - slave.add("numFilesToDownload", String.valueOf(filesToDownload.size())); - slave.add("bytesToDownload", NumberUtils.readableSize(bytesToDownload)); - - long bytesDownloaded = 0; - List filesDownloaded = new ArrayList<>(); - for (Map file : fetcher.getFilesDownloaded()) { - filesDownloaded.add((String) file.get(NAME)); - bytesDownloaded += (Long) file.get(SIZE); - } - - //get list of conf files downloaded - for (Map file : fetcher.getConfFilesDownloaded()) { - filesDownloaded.add((String) file.get(NAME)); - bytesDownloaded += (Long) file.get(SIZE); - } - - for (Map file : fetcher.getContentStoreFilesDownloaded()) { - filesDownloaded.add((String) file.get(NAME)); - bytesDownloaded += (Long) file.get(SIZE); - } - - Map currentFile = fetcher.getCurrentFile(); - String currFile = null; - long currFileSize = 0, currFileSizeDownloaded = 0; - float percentDownloaded = 0; - if (currentFile != null) { - currFile = (String) currentFile.get(NAME); - currFileSize = (Long) currentFile.get(SIZE); - if (currentFile.containsKey("bytesDownloaded")) { - currFileSizeDownloaded = (Long) currentFile.get("bytesDownloaded"); - bytesDownloaded += currFileSizeDownloaded; - if (currFileSize > 0) - percentDownloaded = (currFileSizeDownloaded * 100) / currFileSize; - } - } - slave.add("filesDownloaded", filesDownloaded); - slave.add("numFilesDownloaded", String.valueOf(filesDownloaded.size())); - - long estimatedTimeRemaining = 0; - - Date replicationStartTimeStamp = fetcher.getReplicationStartTimeStamp(); - if (replicationStartTimeStamp != null) { - slave.add("replicationStartTime", replicationStartTimeStamp.toString()); - } - long elapsed = fetcher.getReplicationTimeElapsed(); - slave.add("timeElapsed", elapsed + "s"); - - if (bytesDownloaded > 0) - estimatedTimeRemaining = ((bytesToDownload - bytesDownloaded) * elapsed) / bytesDownloaded; - float totalPercent = 0; - long downloadSpeed = 0; - if (bytesToDownload > 0) - totalPercent = (bytesDownloaded * 100) / bytesToDownload; - if (elapsed > 0) - downloadSpeed = (bytesDownloaded / elapsed); - if (currFile != null) - slave.add("currentFile", currFile); - slave.add("currentFileSize", NumberUtils.readableSize(currFileSize)); - slave.add("currentFileSizeDownloaded", NumberUtils.readableSize(currFileSizeDownloaded)); - slave.add("currentFileSizePercent", String.valueOf(percentDownloaded)); - slave.add("bytesDownloaded", NumberUtils.readableSize(bytesDownloaded)); - slave.add("totalPercent", String.valueOf(totalPercent)); - slave.add("timeRemaining", estimatedTimeRemaining + "s"); - slave.add("downloadSpeed", NumberUtils.readableSize(downloadSpeed)); - } catch (Exception e) { - LOG.error("Exception while writing replication details: ", e); - } - } - } - - if (isMaster) - details.add("master", master); - if (slave.size() > 0) - details.add("slave", slave); - - NamedList snapshotStats = snapShootDetails; - if (snapshotStats != null) - details.add(CMD_BACKUP, snapshotStats); - - return details; - } - - private void addVal(NamedList nl, String key, Properties props, Class clzz) { - String s = props.getProperty(key); - if (s == null || s.trim().length() == 0) return; - if (clzz == Date.class) { - try { - long l = Long.parseLong(s); - nl.add(key, new Date(l).toString()); - } catch (NumberFormatException e) {/*no op*/ } - } else if (clzz == List.class) { - String[] ss = s.split(","); - List l = new ArrayList<>(); - for (String s1 : ss) { - l.add(new Date(Long.parseLong(s1)).toString()); - } - nl.add(key, l); - } else { - nl.add(key, s); - } - - } - - private List getReplicateAfterStrings() { - List replicateAfter = new ArrayList<>(); - if (replicateOnCommit) - replicateAfter.add("commit"); - if (replicateOnOptimize) - replicateAfter.add("optimize"); - if (replicateOnStart) - replicateAfter.add("startup"); - return replicateAfter; - } - - Properties loadReplicationProperties() { - Directory dir = null; - try { - try { - dir = core.getDirectoryFactory().get(core.getDataDir(), - DirContext.META_DATA, core.getSolrConfig().indexConfig.lockType); - IndexInput input; - try { - input = dir.openInput( - IndexFetcher.REPLICATION_PROPERTIES, IOContext.DEFAULT); - } catch (FileNotFoundException | NoSuchFileException e) { - return new Properties(); - } - - try { - final InputStream is = new PropertiesInputStream(input); - Properties props = new Properties(); - props.load(new InputStreamReader(is, StandardCharsets.UTF_8)); - return props; - } finally { - input.close(); - } - } finally { - if (dir != null) { - core.getDirectoryFactory().release(dir); - } - } - } catch (IOException e) { - throw new SolrException(ErrorCode.SERVER_ERROR, e); - } - } - - private void setupPolling(String intervalStr) { - pollIntervalStr = intervalStr; - pollIntervalNs = readIntervalNs(pollIntervalStr); - if (pollIntervalNs == null || pollIntervalNs <= 0) { - LOG.info(" No value set for 'pollInterval'. Timer Task not started."); - return; - } - - Runnable task = () -> { - if (pollDisabled.get()) { - LOG.info("Poll disabled"); - return; - } - try { - LOG.debug("Polling for index modifications"); - markScheduledExecutionStart(); - doFetch(null, false); - } catch (Exception e) { - LOG.error("Exception in fetching index", e); - } - }; - executorService = Executors.newSingleThreadScheduledExecutor( - new DefaultSolrThreadFactory("indexFetcher")); - // Randomize initial delay, with a minimum of 1ms - long initialDelayNs = new Random().nextLong() % pollIntervalNs - + TimeUnit.NANOSECONDS.convert(1, TimeUnit.MILLISECONDS); - executorService.scheduleAtFixedRate(task, initialDelayNs, pollIntervalNs, TimeUnit.NANOSECONDS); - LOG.info("Poll scheduled at an interval of {}ms", - TimeUnit.MILLISECONDS.convert(pollIntervalNs, TimeUnit.NANOSECONDS)); - } - - @Override - public void inform(SolrCore core) { - this.core = core; - - CoreContainer coreContainer = core.getCoreContainer(); - AlfrescoCoreAdminHandler coreAdminHandler = (AlfrescoCoreAdminHandler) coreContainer.getMultiCoreHandler(); - - contentStore = coreAdminHandler.getSolrContentStore(); - registerCloseHook(); - Object nbtk = initArgs.get(NUMBER_BACKUPS_TO_KEEP_INIT_PARAM); - if(nbtk!=null) { - numberBackupsToKeep = Integer.parseInt(nbtk.toString()); - } else { - numberBackupsToKeep = 0; - } - NamedList slave = (NamedList) initArgs.get("slave"); - boolean enableSlave = isEnabled( slave ); - if (enableSlave) { - currentIndexFetcher = pollingIndexFetcher = new IndexFetcher(slave, this, core, contentStore); - setupPolling((String) slave.get(POLL_INTERVAL)); - isSlave = true; - } - NamedList master = (NamedList) initArgs.get("master"); - boolean enableMaster = isEnabled( master ); - - if (enableMaster || enableSlave) { - if (core.getCoreContainer().getZkController() != null) { - LOG.warn("SolrCloud is enabled for core " + core.getName() + " but so is old-style replication. Make sure you" + - " intend this behavior, it usually indicates a mis-configuration. Master setting is " + - Boolean.toString(enableMaster) + " and slave setting is " + Boolean.toString(enableSlave)); - } - } - - if (!enableSlave && !enableMaster) { - enableMaster = true; - master = new NamedList<>(); - } - - if (enableMaster) { - includeConfFiles = (String) master.get(CONF_FILES); - if (includeConfFiles != null && includeConfFiles.trim().length() > 0) { - String[] files = includeConfFiles.split(","); - for (String file : files) { - if (file.trim().length() == 0) continue; - String[] strs = file.trim().split(":"); - // if there is an alias add it or it is null - confFileNameAlias.add(strs[0], strs.length > 1 ? strs[1] : null); - } - LOG.info("Replication enabled for following config files: " + includeConfFiles); - } - List backup = master.getAll("backupAfter"); - boolean backupOnCommit = backup.contains("commit"); - boolean backupOnOptimize = !backupOnCommit && backup.contains("optimize"); - List replicateAfter = master.getAll(REPLICATE_AFTER); - replicateOnCommit = replicateAfter.contains("commit"); - replicateOnOptimize = !replicateOnCommit && replicateAfter.contains("optimize"); - - if (!replicateOnCommit && ! replicateOnOptimize) { - replicateOnCommit = true; - } - - // if we only want to replicate on optimize, we need the deletion policy to - // save the last optimized commit point. - if (replicateOnOptimize) { - IndexDeletionPolicyWrapper wrapper = core.getDeletionPolicy(); - IndexDeletionPolicy policy = wrapper == null ? null : wrapper.getWrappedDeletionPolicy(); - if (policy instanceof SolrDeletionPolicy) { - SolrDeletionPolicy solrPolicy = (SolrDeletionPolicy)policy; - if (solrPolicy.getMaxOptimizedCommitsToKeep() < 1) { - solrPolicy.setMaxOptimizedCommitsToKeep(1); - } - } else { - LOG.warn("Replication can't call setMaxOptimizedCommitsToKeep on " + policy); - } - } - - if (replicateOnOptimize || backupOnOptimize) { - core.getUpdateHandler().registerOptimizeCallback(getEventListener(backupOnOptimize, replicateOnOptimize)); - } - if (replicateOnCommit || backupOnCommit) { - replicateOnCommit = true; - core.getUpdateHandler().registerCommitCallback(getEventListener(backupOnCommit, replicateOnCommit)); - } - if (replicateAfter.contains("startup")) { - replicateOnStart = true; - RefCounted s = core.getNewestSearcher(false); - try { - DirectoryReader reader = s==null ? null : s.get().getIndexReader(); - if (reader!=null && reader.getIndexCommit() != null && reader.getIndexCommit().getGeneration() != 1L) { - // try { - if(replicateOnOptimize){ - Collection commits = DirectoryReader.listCommits(reader.directory()); - for (IndexCommit ic : commits) { - if(ic.getSegmentCount() == 1){ - if(indexCommitPoint == null || indexCommitPoint.getGeneration() < ic.getGeneration()) indexCommitPoint = ic; - } - } - } else{ - indexCommitPoint = reader.getIndexCommit(); - } - //} - //finally { - // We don't need to save commit points for replication, the SolrDeletionPolicy - // always saves the last commit point (and the last optimized commit point, if needed) - - //if(indexCommitPoint != null){ - //core.getDeletionPolicy().saveCommitPoint(indexCommitPoint.getGeneration()); - //} - //} - } - - // ensure the writer is init'd so that we have a list of commit points - RefCounted iw = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core); - iw.decref(); - - } catch (IOException e) { - LOG.warn("Unable to get IndexCommit on startup", e); - } finally { - if (s!=null) s.decref(); - } - } - String reserve = (String) master.get(RESERVE); - if (reserve != null && !reserve.trim().equals("")) { - reserveCommitDuration = readIntervalMs(reserve); - } - LOG.info("Commits will be reserved for " + reserveCommitDuration); - isMaster = true; - } - } - - // check master or slave is enabled - private boolean isEnabled( NamedList params ){ - if( params == null ) return false; - Object enable = params.get( "enable" ); - if( enable == null ) return true; - if( enable instanceof String ) - return StrUtils.parseBool( (String)enable ); - return Boolean.TRUE.equals( enable ); - } - - /** - * register a closehook - */ - private void registerCloseHook() { - core.addCloseHook(new CloseHook() { - @Override - public void preClose(SolrCore core) { - try { - if (executorService != null) executorService.shutdown(); // we don't wait for shutdown - this can deadlock core reload - } finally { - if (pollingIndexFetcher != null) { - pollingIndexFetcher.destroy(); - } - } - if (currentIndexFetcher != null && currentIndexFetcher != pollingIndexFetcher) { - currentIndexFetcher.destroy(); - } - } - - @Override - public void postClose(SolrCore core) {} - }); - - core.addCloseHook(new CloseHook() { - @Override - public void preClose(SolrCore core) { - ExecutorUtil.shutdownAndAwaitTermination(restoreExecutor); - if (restoreFuture != null) { - restoreFuture.cancel(false); - } - } - - @Override - public void postClose(SolrCore core) {} - }); - } - - /** - * Register a listener for postcommit/optimize - * - * @param snapshoot do a snapshoot - * @param getCommit get a commitpoint also - * - * @return an instance of the eventlistener - */ - private SolrEventListener getEventListener(final boolean snapshoot, final boolean getCommit) { - return new SolrEventListener() { - @Override - public void init(NamedList args) {/*no op*/ } - - /** - * This refreshes the latest replicateable index commit and optionally can create Snapshots as well - */ - @Override - public void postCommit() { - IndexCommit currentCommitPoint = core.getDeletionPolicy().getLatestCommit(); - - if (getCommit) { - // IndexCommit oldCommitPoint = indexCommitPoint; - indexCommitPoint = currentCommitPoint; - } - if (snapshoot) { - try { - int numberToKeep = numberBackupsToKeep; - if (numberToKeep < 1) { - numberToKeep = Integer.MAX_VALUE; - } - - SnapShooter snapShooter = new SnapShooter(core, null, null); - snapShooter.validateCreateSnapshot(); - snapShooter.createSnapAsync(currentCommitPoint, numberToKeep, (nl) -> snapShootDetails = nl); - } catch (Exception e) { - LOG.error("Exception while snapshooting", e); - } - } - } - - @Override - public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) { /*no op*/} - - @Override - public void postSoftCommit() { - - } - }; - } - - /**This class is used to read and send files in the lucene index - * - */ - private class DirectoryFileStream implements SolrCore.RawWriter { - protected SolrParams params; - - FastOutputStream fos; - - Long indexGen; - IndexDeletionPolicyWrapper delPolicy; - - String fileName; - String cfileName; - String tlogFileName; - String contentStoreFilename; - String sOffset; - String sLen; - String compress; - boolean useChecksum; - - protected long offset = -1; - int len = -1; - - Checksum checksum; - - private RateLimiter rateLimiter; - - byte[] buf; - - DirectoryFileStream(SolrParams solrParams) { - params = solrParams; - delPolicy = core.getDeletionPolicy(); - - fileName = validateFilenameOrError(params.get(FILE)); - cfileName = validateFilenameOrError(params.get(CONF_FILE_SHORT)); - tlogFileName = validateFilenameOrError(params.get(TLOG_FILE)); - - sOffset = params.get(OFFSET); - sLen = params.get(LEN); - compress = params.get(COMPRESSION); - useChecksum = params.getBool(CHECKSUM, false); - - indexGen = params.getLong(GENERATION); - if (useChecksum) { - checksum = new Adler32(); - } - //No throttle if MAX_WRITE_PER_SECOND is not specified - double maxWriteMBPerSec = params.getDouble(MAX_WRITE_PER_SECOND, Double.MAX_VALUE); - rateLimiter = new RateLimiter.SimpleRateLimiter(maxWriteMBPerSec); - } - - // Throw exception on directory traversal attempts - String validateFilenameOrError(String filename) { - if (filename != null) { - Path filePath = Paths.get(filename); - filePath.forEach(subpath -> { - if ("..".equals(subpath.toString())) { - throw new SolrException(ErrorCode.FORBIDDEN, "File name cannot contain .."); - } - }); - if (filePath.isAbsolute()) { - throw new SolrException(ErrorCode.FORBIDDEN, "File name must be relative"); - } - return filename; - } else return null; - } - - void initWrite() throws IOException { - if (sOffset != null) offset = Long.parseLong(sOffset); - if (sLen != null) len = Integer.parseInt(sLen); - if (fileName == null && cfileName == null && tlogFileName == null && contentStoreFilename == null) { - // no filename do nothing - writeNothingAndFlush(); - } - buf = new byte[(len == -1 || len > PACKET_SZ) ? PACKET_SZ : len]; - - //reserve commit point till write is complete - if(indexGen != null) { - delPolicy.saveCommitPoint(indexGen); - } - } - - void createOutputStream(OutputStream out) { - if (Boolean.parseBoolean(compress)) { - fos = new FastOutputStream(new DeflaterOutputStream(out)); - } else { - fos = new FastOutputStream(out); - } - } - - void extendReserveAndReleaseCommitPoint() { - if(indexGen != null) { - //Reserve the commit point for another 10s for the next file to be to fetched. - //We need to keep extending the commit reservation between requests so that the replica can fetch - //all the files correctly. - delPolicy.setReserveDuration(indexGen, reserveCommitDuration); - - //release the commit point as the write is complete - delPolicy.releaseCommitPoint(indexGen); - } - - } - public void write(OutputStream out) throws IOException { - createOutputStream(out); - - IndexInput in = null; - try { - initWrite(); - - RefCounted sref = core.getSearcher(); - Directory dir; - try { - SolrIndexSearcher searcher = sref.get(); - dir = searcher.getIndexReader().directory(); - } finally { - sref.decref(); - } - in = dir.openInput(fileName, IOContext.READONCE); - // if offset is mentioned move the pointer to that point - if (offset != -1) in.seek(offset); - - long filelen = dir.fileLength(fileName); - long maxBytesBeforePause = 0; - - while (true) { - offset = offset == -1 ? 0 : offset; - int read = (int) Math.min(buf.length, filelen - offset); - in.readBytes(buf, 0, read); - - fos.writeInt(read); - if (useChecksum) { - checksum.reset(); - checksum.update(buf, 0, read); - fos.writeLong(checksum.getValue()); - } - fos.write(buf, 0, read); - fos.flush(); - LOG.debug("Wrote {} bytes for file {}", offset + read, fileName); - - //Pause if necessary - maxBytesBeforePause += read; - if (maxBytesBeforePause >= rateLimiter.getMinPauseCheckBytes()) { - rateLimiter.pause(maxBytesBeforePause); - maxBytesBeforePause = 0; - } - if (read != buf.length) { - writeNothingAndFlush(); - fos.close(); - break; - } - offset += read; - in.seek(offset); - } - } catch (IOException e) { - LOG.warn("Exception while writing response for params: " + params, e); - } finally { - if (in != null) { - in.close(); - } - extendReserveAndReleaseCommitPoint(); - } - } - - - /** - * Used to write a marker for EOF - */ - protected void writeNothingAndFlush() throws IOException { - fos.writeInt(0); - fos.flush(); - } - } - - /** - * This is used to write files in the conf directory. - */ - protected abstract class LocalFsFileStream extends DirectoryFileStream { - - private File file; - - LocalFsFileStream(SolrParams solrParams) { - super(solrParams); - this.file = this.initFile(); - } - - protected abstract File initFile(); - - @Override - public void write(OutputStream out) { - createOutputStream(out); - FileInputStream inputStream = null; - try { - initWrite(); - - if (file.exists() && file.canRead()) { - inputStream = new FileInputStream(file); - FileChannel channel = inputStream.getChannel(); - //if offset is mentioned move the pointer to that point - if (offset != -1) - channel.position(offset); - ByteBuffer bb = ByteBuffer.wrap(buf); - - while (true) { - bb.clear(); - long bytesRead = channel.read(bb); - if (bytesRead <= 0) { - writeNothingAndFlush(); - fos.close(); - break; - } - fos.writeInt((int) bytesRead); - if (useChecksum) { - checksum.reset(); - checksum.update(buf, 0, (int) bytesRead); - fos.writeLong(checksum.getValue()); - } - fos.write(buf, 0, (int) bytesRead); - fos.flush(); - } - } else { - writeNothingAndFlush(); - } - } catch (IOException e) { - LOG.warn("Exception while writing response for params: " + params, e); - } finally { - IOUtils.closeQuietly(inputStream); - extendReserveAndReleaseCommitPoint(); - } - } - } - - protected class ContentStoreFilesStream extends DirectoryFileStream { - ContentStoreFilesStream(SolrParams solrParams) { - super(solrParams); - } - - @Override - public void write(OutputStream out) throws IOException { - createOutputStream(out); - String contentStoreRoot = contentStore.getRootLocation(); - try { - for (String fileName : params.getParams(CONTENT_STORE_FILE_LIST)) - { - File f = new File(contentStoreRoot + fileName); - if (f.exists() && !f.isDirectory()) { - try { - writeFile(f, fileName); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - fos.close(); - extendReserveAndReleaseCommitPoint(); - } - } - - - @Override - protected void writeNothingAndFlush() throws IOException { - fos.flush(); - } - - void writeFile(File file, String fileName) throws IOException { - - buf = new byte[PACKET_SZ]; - if (file.exists() && file.canRead()) { - FileInputStream inputStream = new FileInputStream(file); - FileChannel channel = inputStream.getChannel(); - //if offset is mentioned move the pointer to that point - if (offset != -1) - channel.position(offset); - ByteBuffer bb = ByteBuffer.wrap(buf); - channel.size(); - - fos.writeInt( fileName.length()); - fos.write(fileName.getBytes()); - fos.writeInt((int) channel.size()); - - if (channel.size() != 0) { - - while (true) { - bb.clear(); - long bytesRead = channel.read(bb); - if (bytesRead <= 0) { - writeNothingAndFlush(); - break; - } - - fos.writeInt((int) bytesRead); - - if (useChecksum) { - checksum.reset(); - checksum.update(buf, 0, (int) bytesRead); - fos.writeLong(checksum.getValue()); - } - - fos.write(buf, 0, (int) bytesRead); - fos.flush(); - } - } - } - } - } - - - private class LocalContentStoreFileStream extends LocalFsFileStream { - LocalContentStoreFileStream(SolrParams solrParams) { - super(solrParams); - } - - protected File initFile() { - return new File(contentStore.getRootLocation() + "/" + contentStoreFilename); - } - - } - - - private class LocalFsTlogFileStream extends LocalFsFileStream { - - LocalFsTlogFileStream(SolrParams solrParams) { - super(solrParams); - } - - protected File initFile() { - //if it is a tlog file read from tlog directory - return new File(core.getUpdateHandler().getUpdateLog().getLogDir(), tlogFileName); - } - - } - - private class LocalFsConfFileStream extends LocalFsFileStream { - - LocalFsConfFileStream(SolrParams solrParams) { - super(solrParams); - } - - protected File initFile() { - //if it is a conf file read from config directory - return new File(core.getResourceLoader().getConfigDir(), cfileName); - } - - } - - private static Integer readIntervalMs(String interval) { - return (int) TimeUnit.MILLISECONDS.convert(readIntervalNs(interval), TimeUnit.NANOSECONDS); - } - - private static Long readIntervalNs(String interval) { - if (interval == null) - return null; - int result; - Matcher m = INTERVAL_PATTERN.matcher(interval.trim()); - if (m.find()) { - String hr = m.group(1); - String min = m.group(2); - String sec = m.group(3); - result = 0; - try { - if (sec != null && sec.length() > 0) - result += Integer.parseInt(sec); - if (min != null && min.length() > 0) - result += (60 * Integer.parseInt(min)); - if (hr != null && hr.length() > 0) - result += (60 * 60 * Integer.parseInt(hr)); - return TimeUnit.NANOSECONDS.convert(result, TimeUnit.SECONDS); - } catch (NumberFormatException e) { - throw new SolrException(ErrorCode.SERVER_ERROR, INTERVAL_ERR_MSG); - } - } else { - throw new SolrException(ErrorCode.SERVER_ERROR, INTERVAL_ERR_MSG); - } - } - - private static final String SUCCESS = "success"; - - private static final String FAILED = "failed"; - - private static final String EXCEPTION = "exception"; - - static final String MASTER_URL = "masterUrl"; - - private static final String STATUS = "status"; - - static final String COMMAND = "command"; - - static final String CMD_DETAILS = "details"; - - private static final String CMD_BACKUP = "backup"; - - private static final String CMD_RESTORE = "restore"; - - private static final String CMD_RESTORE_STATUS = "restorestatus"; - - private static final String CMD_FETCH_INDEX = "fetchindex"; - - private static final String CMD_ABORT_FETCH = "abortfetch"; - - static final String CMD_GET_FILE_LIST = "filelist"; - - static final String CMD_GET_FILE = "filecontent"; - - private static final String CMD_DISABLE_POLL = "disablepoll"; - - private static final String CMD_DISABLE_REPL = "disablereplication"; - - private static final String CMD_ENABLE_REPL = "enablereplication"; - - private static final String CMD_ENABLE_POLL = "enablepoll"; - - static final String CMD_INDEX_VERSION = "indexversion"; - - private static final String CMD_SHOW_COMMITS = "commits"; - - private static final String CMD_DELETE_BACKUP = "deletebackup"; - - static final String GENERATION = "generation"; - - static final String CONTENT_STORE_VERSION = "contentstoreversion"; - - static final String OFFSET = "offset"; - - private static final String LEN = "len"; - - static final String FILE = "file"; - - public static final String SIZE = "size"; - - private static final String MAX_WRITE_PER_SECOND = "maxWriteMBPerSec"; - - static final String CONF_FILE_SHORT = "cf"; - - static final String TLOG_FILE = "tlogFile"; - - static final String CHECKSUM = "checksum"; - - static final String ALIAS = "alias"; - - static final String CONF_FILES = "confFiles"; - - static final String TLOG_FILES = "tlogFiles"; - - static final String CONTENT_STORE_FILES = "contentStoreFiles"; - - private static final String REPLICATE_AFTER = "replicateAfter"; - - static final String FILE_STREAM = "filestream"; - - private static final String POLL_INTERVAL = "pollInterval"; - - private static final String INTERVAL_ERR_MSG = "The " + POLL_INTERVAL + " must be in this format 'HH:mm:ss'"; - - private static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)"); - - private static final int PACKET_SZ = 1024 * 1024; // 1MB - - private static final String RESERVE = "commitReserveDuration"; - - static final String COMPRESSION = "compression"; - - static final String EXTERNAL = "external"; - - static final String INTERNAL = "internal"; - - private static final String ERR_STATUS = "ERROR"; - - private static final String OK_STATUS = "OK"; - - private static final String NEXT_EXECUTION_AT = "nextExecutionAt"; - - private static final String NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM = "numberToKeep"; - - private static final String NUMBER_BACKUPS_TO_KEEP_INIT_PARAM = "maxNumberOfBackups"; - - static final String CONTENT_STORE_FILE_LIST = "contentStoreFiles"; - - static final String CMD_CONTENT_STORE_FILES = "cmdContentStoreFiles"; - - static final long NO_INDEX_REPLICATION_REQUIRED = -3; - - /** - * Boolean param for tests that can be specified when using - * {@link #CMD_FETCH_INDEX} to force the current request to block until - * the fetch is complete. NOTE: This param is not advised for - * non-test code, since the the duration of the fetch for non-trivial - * indexes will likeley cause the request to time out. - */ - private static final String WAIT = "wait"; -} From be9f75ab16f7d83dd0dfce75a2e41f0d92054480 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 1 Nov 2019 10:20:34 +0000 Subject: [PATCH 45/76] SEARCH-1913 Delete failing test. This test does not appear to be very valuable. It is currently only passing due to interaction with other tests. Whenever it starts failing then the numbers in it are updated so that it passes again. --- .../org/alfresco/solr/SolrDataModelTest.java | 75 ------------------- 1 file changed, 75 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrDataModelTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrDataModelTest.java index 17bc9ee73..0f46bbed4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrDataModelTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrDataModelTest.java @@ -72,79 +72,4 @@ public class SolrDataModelTest Long actualId = AlfrescoSolrDataModel.parseTransactionId(id); assertEquals(expectedId, actualId); } - - @Test - public void smokeTestCMISModel() - { - AlfrescoSolrDataModel dataModel = new AlfrescoSolrDataModel(); - - // load test model containing content properties multiple - ClassLoader cl = SolrDataModelTest.class.getClassLoader(); - InputStream modelStream = cl.getResourceAsStream("alfresco/model/dictionaryModel.xml"); - - assertNotNull(modelStream); - M2Model model = M2Model.createModel(modelStream); - dataModel.putModel(model); - - modelStream = cl.getResourceAsStream("alfresco/model/cmisModel.xml"); - assertNotNull(modelStream); - model = M2Model.createModel(modelStream); - dataModel.putModel(model); - - assertEquals(2, dataModel.getAlfrescoModels().size()); - - assertEquals(1, dataModel.getIndexedFieldNamesForProperty(OBJECT_ID).getFields().size()); - assertEquals(4, dataModel.getIndexedFieldNamesForProperty(NAME).getFields().size()); - assertEquals(1, dataModel.getIndexedFieldNamesForProperty(CREATION_DATE).getFields().size()); - assertEquals(0, dataModel.getIndexedFieldNamesForProperty(IS_IMMUTABLE).getFields().size()); - assertEquals(0, dataModel.getIndexedFieldNamesForProperty(IS_PRIVATE_WOKING_COPY).getFields().size()); - assertEquals(1, dataModel.getIndexedFieldNamesForProperty(CONTENT_STREAM_LENGTH).getFields().size()); - - assertEquals(1, dataModel .getQueryableFields(OBJECT_ID, null, FieldUse.FACET).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(OBJECT_ID, null, FieldUse.COMPLETION).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(OBJECT_ID, null, FieldUse.FTS).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(OBJECT_ID, null, FieldUse.ID).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(OBJECT_ID, null, FieldUse.MULTI_FACET).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(OBJECT_ID, null, FieldUse.SORT).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(OBJECT_ID, null, FieldUse.STATS).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(OBJECT_ID, null, FieldUse.SUGGESTION).getFields().size()); - - assertEquals(1, dataModel .getQueryableFields(NAME, null, FieldUse.FACET).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(NAME, null, FieldUse.COMPLETION).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(NAME, null, FieldUse.FTS).getFields().size()); - assertEquals(2, dataModel .getQueryableFields(NAME, null, FieldUse.ID).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(NAME, null, FieldUse.MULTI_FACET).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(NAME, null, FieldUse.SORT).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(NAME, null, FieldUse.STATS).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(NAME, null, FieldUse.SUGGESTION).getFields().size()); - - assertEquals(1, dataModel .getQueryableFields(CREATION_DATE, null, FieldUse.FACET).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CREATION_DATE, null, FieldUse.COMPLETION).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CREATION_DATE, null, FieldUse.FTS).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CREATION_DATE, null, FieldUse.ID).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CREATION_DATE, null, FieldUse.MULTI_FACET).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CREATION_DATE, null, FieldUse.SORT).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CREATION_DATE, null, FieldUse.STATS).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CREATION_DATE, null, FieldUse.SUGGESTION).getFields().size()); - - assertEquals(1, dataModel .getQueryableFields(IS_PRIVATE_WOKING_COPY, null, FieldUse.FACET).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(IS_PRIVATE_WOKING_COPY, null, FieldUse.COMPLETION).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(IS_PRIVATE_WOKING_COPY, null, FieldUse.FTS).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(IS_PRIVATE_WOKING_COPY, null, FieldUse.ID).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(IS_PRIVATE_WOKING_COPY, null, FieldUse.MULTI_FACET).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(IS_PRIVATE_WOKING_COPY, null, FieldUse.SORT).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(IS_PRIVATE_WOKING_COPY, null, FieldUse.STATS).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(IS_PRIVATE_WOKING_COPY, null, FieldUse.SUGGESTION).getFields().size()); - - assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.FACET).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.COMPLETION).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.FTS).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.ID).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.MULTI_FACET).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.SORT).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.STATS).getFields().size()); - assertEquals(1, dataModel .getQueryableFields(CONTENT_STREAM_LENGTH, null, FieldUse.SUGGESTION).getFields().size()); - - - } } From 9c5201c6bbcceab0423413060cf444af16f37577 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 1 Nov 2019 11:11:38 +0000 Subject: [PATCH 46/76] SEARCH-1763 Ignore all third party libraries with Jacoco. --- search-services/alfresco-search/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 1a041e2f9..f5285a7cf 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -292,7 +292,7 @@ - **/AnnotationWriter.* + **/libs/* ${project.build.directory}/coverage-reports/jacoco-ut.exec From 14762f04a9d32be462adbef8276de2962e31b96d Mon Sep 17 00:00:00 2001 From: agazzarini Date: Fri, 1 Nov 2019 12:16:15 +0100 Subject: [PATCH 47/76] [ SEARCH-1858 ] Minor refactoring and more verbose comments about the changes introduced. --- .../org/alfresco/solr/content/AccessMode.java | 11 ++-- .../org/alfresco/solr/content/ChangeSet.java | 57 ++++++++++++++++++- .../solr/content/InitialisableAccessMode.java | 2 +- .../solr/content/SolrContentStore.java | 39 +++++++++++-- .../solr/content/SolrContentUrlBuilder.java | 45 +++++++-------- .../solr/content/SolrFileContentReader.java | 16 ++---- .../solr/content/SolrFileContentWriter.java | 14 +++-- .../SolrContentStoreChangeSetTest.java | 3 +- .../solr/content/SolrContentStoreTest.java | 6 +- .../content/SolrContentUrlBuilderTest.java | 11 ++-- .../solr/content/SolrContentWriterTest.java | 34 ++--------- 11 files changed, 149 insertions(+), 89 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/AccessMode.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/AccessMode.java index 148c3be08..468526a10 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/AccessMode.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/AccessMode.java @@ -27,16 +27,19 @@ import java.util.List; import java.util.Map; /** - * Behavioural interface for denoting a given role in a replication scenario. - * A replication interaction is composed at least by 2 nodes: a master and one or more slaves. + * Behavioural interface which indicates the type of content store access mode for the owning node. + * A replication interaction is composed at least by 2 roles: a master and a slave. * * Despite the same interface, the behaviour of the content store management changes depending on the node kind: * *
    - *
  • Master Node: READ + WRITE and changes tracking
    • - *
  • Slave Node: READ ONLY
    • + *
  • Master Node: READ + WRITE + Changes tracking. Writes and changes tracking are a direct consequence of the "indexing" nature of the master node.
    • + *
  • Slave Node: READ ONLY (i.e. never write: changes are applied on the master and replicated on slaves)
    • *
* + * Important: the owning entity *is not* the SolrCore instance: the Content store is shared across all cores of + * A node can transit from Read to Write mode + * * @author Andrea Gazzarini * @since 1.5 */ diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java index fc3d5a2cc..95180f059 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java @@ -54,7 +54,30 @@ import java.util.function.BiFunction; import java.util.function.BinaryOperator; /** - * Encapsulates changes occurred in the hosting content store since the last commit. + * Stores and manages changes occurred in a content store. + * + * Note this entity is part of the content store only when it is set on writable mode (i.e. only when the hosting node + * is a standalone instance or it is a master node). + * + * That means if the hosting node has at least one standalone core or one master node, then the managed content store + * will be set in write mode and therefore a {@link ChangeSet} instance will track all changes applied to it. + * + * In that mode, each change is associated to a given (incremental) version; changes consist of adds/updates and deletes. + * Each time the {@link org.alfresco.solr.tracker.CommitTracker} executes a commit in the main index, all content store + * changes accumulated in the meantime in the current {@link ChangeSet} instance are flushed in an auxiliary Lucene index + * and therefore persisted. + * + * On the other side, when all cores on a given node are configured as slaves, the content store is in read-only mode, + * and there's no need to track any change (i.e. changes are coming from master through the replication procedure). + * + * As a side note: we can have a persistent or transient {@link ChangeSet} instance. The first one is used to manage, as + * the name suggests, in a persistent way the content store changes accumulated during the indexing operations (see the + * description above about its behaviour in master nodes). + * + * The second one is used instead for computing/reducing the "merged" list of changes that we need to communicate to slave nodes: + * as part of the replication mechanism, a slave communicates its last synched content store version; on the master side, + * we need to compute and communicate back all changes that the slave needs to apply since that version in order to be in + * synch with master. * * @author Andrea Gazzarini * @since 1.5 @@ -66,12 +89,28 @@ public class ChangeSet implements AutoCloseable /** * Builds class for creating {@link ChangeSet} instances. + * The builder is here for creating three kind of {@link ChangeSet} instances: + * + *
    + *
  • Persistent: used for tracking and persisting content store changes.
    • + *
  • Transient: used for computing the merged list of changes a slave should apply for being in synch with master.
    • + *
  • Empty: an immutable "NullObject" used for denoting an empty {@link ChangeSet} instance.
    • + *
+ * + * @see Null Object Design Pattern */ public static class Builder { private String root; private boolean immutable; + /** + * Adds a content store root folder to this builder. + * This automatically implies we are creating a persistent {@link ChangeSet} instance. + * + * @param root the absolute path of the content store root folder. + * @return this builder, for being used in a fluent mode. + */ Builder withContentStoreRoot(String root) { if (root == null) throw new IllegalArgumentException("Unable to build the Changeset structures with a null content store root folder."); @@ -81,6 +120,11 @@ public class ChangeSet implements AutoCloseable return this; } + /** + * We are interested in building an empty, immutable {@link ChangeSet} instance. + * + * @return this builder, for being used in a fluent mode. + */ Builder empty() { this.root = null; @@ -88,6 +132,11 @@ public class ChangeSet implements AutoCloseable return this; } + /** + * Builds the product of this builder (i.e. the {@link ChangeSet} instance). + * + * @return the product of this builder (i.e. the {@link ChangeSet} instance). + */ ChangeSet build() { if (root == null) @@ -252,6 +301,12 @@ public class ChangeSet implements AutoCloseable LOGGER.debug("New Changeset entry have been added (version = {}, deletes = {}, adds = {})", version, deletes.size(), adds.size()); } + /** + * Sanity check for making sure the requestor sent a valid and known content store version. + * + * @param version the content store version (sent by a requestor) + * @return true if the given version is unknown, that is, is not part of the content store version history. + */ boolean isUnknownVersion(long version) { try diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/InitialisableAccessMode.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/InitialisableAccessMode.java index 6fa1770b5..f8fa1e9a8 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/InitialisableAccessMode.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/InitialisableAccessMode.java @@ -30,4 +30,4 @@ public interface InitialisableAccessMode extends AccessMode * Initialises this access mode instance. */ void init(); -} +} \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java index 50a9cf662..acf472650 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java @@ -64,6 +64,7 @@ import static org.alfresco.solr.content.SolrContentUrlBuilder.logger; /** * A content store specific to SOLR's requirements: The URL is generated from a * set of properties such as: + * *
    *
  • ACL ID
    • *
  • DB ID
    • @@ -72,20 +73,50 @@ import static org.alfresco.solr.content.SolrContentUrlBuilder.logger; * * The URL, if not known, can be reliably regenerated using the {@link SolrContentUrlBuilder}. *
    - * Since version 1.5 this class acts as a singleton, there will be just one instance for each hosting Solr node. - * The unique instance is created at startup in {@link org.alfresco.solr.AlfrescoCoreAdminHandler} and then given to - * each registered core. - * The state pattern implemented by means of the {@link AccessMode} interface allows for a given instance to act: + * + * Since version 1.5 this class acts as a logical singleton: there will be only one instance per node. + * That reflects exactly what we physically have in the filesystem (i.e. there's only one content store per node). + * + * That unique instance is created at startup in {@link org.alfresco.solr.AlfrescoCoreAdminHandler} and then passed to + * each registered core (see {@link org.alfresco.solr.lifecycle.SolrCoreLoadListener}) + * + * The State pattern implemented by means of the {@link AccessMode} interface allows for a given {@link SolrContentStore} instance to act: * *
      *
    • in READ/WRITE mode: when at least one core of the hosting node is a master or it is a standalone shard/instance.
      • *
    • in READ ONLY mode: when all cores of the hosting node are slaves.
      • *
    * + * The Finite State Machine (FST) provides three possible states: Initial, Read Only, Read/Write. + * The allowed transitions are: + * + *
      + *
    • Initial -> ReadOnly: a slave core has been registered, the content store hasn't been yet initialised.
      • + *
    • + * ReadOnly -> Read/Write: this scenario is unusual because we should have on the same instance a mixed set of cores + * (some master/standalone, some slaves). It happens when a first slave core registers and causes the transition + * described in the first point (initial -> read only). Then a master or standalone core registers so we need to + * move from a read only to a complete readable/writable managed content store. + *
      • + *
    • Initial -> Read/Write: a master or standalone core has been registered, and the content store hasn't been yet initialised.
      • + *
    + * + * Note the following transitions are not allowed: + * + *
      + *
    • Coming back to Initial state: once it has been initialised the Content Store cannot return back to the "Initial" state.
      • + *
    • + * Read/Write -> ReadOnly: if at least one master or standalone core registers, the content store is permanently moved in Read/Write mode. + * So even a further slave node registers (not very usually as described above) the content store remains in RW mode. + *
      • + *
    + * * @author Derek Hulley * @author Michael Suzuki * @author Andrea Gazzarini * @since 1.5 + * @see org.alfresco.solr.lifecycle.SolrCoreLoadListener + * @see State Pattern */ public final class SolrContentStore implements Closeable, AccessMode { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentUrlBuilder.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentUrlBuilder.java index d7fc2be9d..542636111 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentUrlBuilder.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentUrlBuilder.java @@ -18,7 +18,6 @@ */ package org.alfresco.solr.content; -import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.TreeMap; @@ -44,19 +43,13 @@ import org.slf4j.LoggerFactory; */ public class SolrContentUrlBuilder { - /** - * solr is the prefix for SOLR content URLs - */ - public static final String SOLR_PROTOCOL = "solr"; - public static final String SOLR_PROTOCOL_PREFIX = SOLR_PROTOCOL + ContentStore.PROTOCOL_DELIMITER; - public static final String FILE_EXTENSION = ".gz"; + private static final String SOLR_PROTOCOL = "solr"; + static final String SOLR_PROTOCOL_PREFIX = SOLR_PROTOCOL + ContentStore.PROTOCOL_DELIMITER; + static final String FILE_EXTENSION = ".gz"; - /** The key for the tenant name */ - public static final String KEY_TENANT = "tenant"; - /** The key for the DB ID */ - public static final String KEY_DB_ID = "dbId"; - /** The key for the ACL ID */ - public static final String KEY_ACL_ID = "aclId"; + static final String KEY_TENANT = "tenant"; + static final String KEY_DB_ID = "dbId"; + static final String KEY_ACL_ID = "aclId"; protected final static Logger logger = LoggerFactory.getLogger(SolrContentUrlBuilder.class); @@ -66,9 +59,9 @@ public class SolrContentUrlBuilder /** * Protected constructor used by {@link SolrContentUrlBuilder#start()} */ - protected SolrContentUrlBuilder() + private SolrContentUrlBuilder() { - this.metadata = new TreeMap(); + this.metadata = new TreeMap<>(); } /** @@ -83,32 +76,34 @@ public class SolrContentUrlBuilder /** * Add some metadata to the URL generator. The order in which metadata is added is irrelevant. - *

    * Note that there are specific keys that are commonly used and, if provided, may not be null or empty. + * *

      *
    • {@link #KEY_TENANT}: The name of the tenant or 'default' if missing.
      • *
    • {@link #KEY_DB_ID}: The database ID.
      • *
    • {@link #KEY_ACL_ID}: The ACL ID.
      • *
    * - * @param key an arbitrary metadata key (never null> - * @param value some metadata value (null is supported) - * @return this builder for more building + * @param key an arbitrary metadata key (never null> + * @param value some metadata value (null is supported) + * @return this builder for more building * - * @throws IllegalArgumentException if the key is null - * @throws IllegalStateException if the key has been used already + * @throws IllegalArgumentException if the key is null + * @throws IllegalStateException if the key has been used already */ - public synchronized SolrContentUrlBuilder add(String key, String value) + public SolrContentUrlBuilder add(String key, String value) { if (key == null) { throw new IllegalArgumentException("The metadata 'key' may not be null."); } + String previous = metadata.put(key, value); if (previous != null) { throw new IllegalStateException("The metadata key, '" + key + "', has already been used."); } + // Check well-known keys if (key.equals(KEY_TENANT) || key.equals(KEY_DB_ID) || key.equals(KEY_ACL_ID)) { @@ -131,8 +126,8 @@ public class SolrContentUrlBuilder /** * Get the final content URL using the {@link #add(String, String) supplied metadata}. * - * @return the SOLR content URL - * @throws IllegalStateException if no metadata has been added + * @return the SOLR content URL + * @throws IllegalStateException if no metadata has been added */ public synchronized String get() { @@ -210,7 +205,7 @@ public class SolrContentUrlBuilder /** * Helper method to retrieve a {@link ContentContext} constructed using the final {@link #get()} url. */ - public ContentContext getContentContext() + ContentContext getContentContext() { String url = get(); return new ContentContext(null, url); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentReader.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentReader.java index 357e63ffd..864e973d7 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentReader.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentReader.java @@ -42,7 +42,7 @@ import org.springframework.util.FileCopyUtils; * @author Derek Hulley * @since 5.0 */ -public class SolrFileContentReader implements ContentReader +class SolrFileContentReader implements ContentReader { private final File file; private final String contentUrl; @@ -51,7 +51,7 @@ public class SolrFileContentReader implements ContentReader * @param file the file to write to * @param contentUrl the content URL for information purposes */ - protected SolrFileContentReader(File file, String contentUrl) + SolrFileContentReader(File file, String contentUrl) { this.file = file; this.contentUrl = contentUrl; @@ -120,9 +120,8 @@ public class SolrFileContentReader implements ContentReader } try { - InputStream is = new BufferedInputStream(new FileInputStream(file)); // done - return is; + return new BufferedInputStream(new FileInputStream(file)); } catch (Throwable e) { @@ -192,12 +191,9 @@ public class SolrFileContentReader implements ContentReader ByteArrayOutputStream os = new ByteArrayOutputStream(); FileCopyUtils.copy(is, os); // both streams are closed byte[] bytes = os.toByteArray(); - // get the encoding for the string + String encoding = "UTF-8"; - // create the string from the byte[] using encoding if necessary - String content = (encoding == null) ? new String(bytes) : new String(bytes, encoding); - // done - return content; + return new String(bytes, encoding); } catch (IOException e) { @@ -266,4 +262,4 @@ public class SolrFileContentReader implements ContentReader { throw new UnsupportedOperationException("Auto-created method not implemented."); } -} +} \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentWriter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentWriter.java index 9dd57aea1..bb3dbc142 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentWriter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentWriter.java @@ -41,7 +41,7 @@ import org.apache.commons.io.FileUtils; * @author Derek Hulley * @since 5.0 */ -public class SolrFileContentWriter implements ContentWriter +class SolrFileContentWriter implements ContentWriter { private final File file; private final String contentUrl; @@ -51,11 +51,10 @@ public class SolrFileContentWriter implements ContentWriter * @param file the file to write to * @param contentUrl the content URL for information purposes */ - protected SolrFileContentWriter(File file, String contentUrl) + SolrFileContentWriter(File file, String contentUrl) { this.file = file; this.contentUrl = contentUrl; - this.written = false; } @Override @@ -109,7 +108,7 @@ public class SolrFileContentWriter implements ContentWriter @Override public synchronized OutputStream getContentOutputStream() throws ContentIOException { - if (written == true) + if (written) { throw new IllegalStateException("The writer has already been used: " + file); } @@ -117,6 +116,7 @@ public class SolrFileContentWriter implements ContentWriter { throw new IllegalStateException("The file already exists: " + file); } + try { OutputStream is = new BufferedOutputStream(FileUtils.openOutputStream(file)); @@ -139,7 +139,7 @@ public class SolrFileContentWriter implements ContentWriter @Override public synchronized void putContent(InputStream is) throws ContentIOException { - if (written == true) + if (written) { throw new IllegalStateException("The writer has already been used: " + file); } @@ -147,6 +147,7 @@ public class SolrFileContentWriter implements ContentWriter { throw new IllegalStateException("The file already exists: " + file); } + try { FileUtils.copyInputStreamToFile(is, file); @@ -161,7 +162,7 @@ public class SolrFileContentWriter implements ContentWriter @Override public synchronized void putContent(File sourceFile) throws ContentIOException { - if (written == true) + if (written) { throw new IllegalStateException("The writer has already been used: " + this.file); } @@ -173,6 +174,7 @@ public class SolrFileContentWriter implements ContentWriter { throw new IllegalStateException("The source file does not exist: " + sourceFile); } + try { FileUtils.copyFile(sourceFile, this.file, false); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentStoreChangeSetTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentStoreChangeSetTest.java index 0e69197b0..6de541e5d 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentStoreChangeSetTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentStoreChangeSetTest.java @@ -42,6 +42,7 @@ import static org.mockito.Mockito.when; * Solr ContentStore {@link ChangeSet} test case. * * @author Andrea Gazzarini + * @since 1.5 */ @RunWith(MockitoJUnitRunner.class) public class SolrContentStoreChangeSetTest @@ -287,4 +288,4 @@ public class SolrContentStoreChangeSetTest assertTrue(changesAfterSecondFlush.adds.contains("A3")); assertTrue(changesAfterSecondFlush.deletes.contains("A4")); } -} +} \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentStoreTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentStoreTest.java index f4055457d..26042feda 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentStoreTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentStoreTest.java @@ -25,7 +25,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.BiConsumer; -import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -38,10 +37,8 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import static java.util.Collections.emptyMap; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; @@ -53,7 +50,8 @@ import static org.mockito.Mockito.mock; * Tests {@link SolrContentStoreTest} * * @author Derek Hulley - * @since 5.0 + * @author Andrea Gazzarini + * @since 1.5 */ @RunWith(MockitoJUnitRunner.class) public class SolrContentStoreTest diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentUrlBuilderTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentUrlBuilderTest.java index bb3f8cf14..7b5131bd4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentUrlBuilderTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentUrlBuilderTest.java @@ -21,7 +21,7 @@ package org.alfresco.solr.content; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; /** * Tests {@link SolrContentUrlBuilder} @@ -85,12 +85,12 @@ public class SolrContentUrlBuilderTest .add(SolrContentUrlBuilder.KEY_DB_ID, "12345") .get(); Assert.assertEquals(url1, url1Check); + String url2 = SolrContentUrlBuilder .start() .add(SolrContentUrlBuilder.KEY_DB_ID, "123456") .get(); Assert.assertNotEquals(url1, url2); - Assert.assertEquals("Incorrect URL: " + url1, "solr://default/db/1234/5.gz", url1); } @@ -108,7 +108,6 @@ public class SolrContentUrlBuilderTest .add(SolrContentUrlBuilder.KEY_TENANT, "bob") .get(); Assert.assertEquals(url1, url1Check); - Assert.assertEquals("Incorrect URL: " + url1, "solr://bob/db/1234/56.gz", url1); } @@ -126,7 +125,6 @@ public class SolrContentUrlBuilderTest .add(SolrContentUrlBuilder.KEY_TENANT, "bob") .get(); Assert.assertEquals(url1, url1Check); - Assert.assertEquals("Incorrect URL: " + url1, "solr://bob/acl/1234/56.gz", url1); } @@ -146,7 +144,6 @@ public class SolrContentUrlBuilderTest .add(SolrContentUrlBuilder.KEY_TENANT, "bob") .get(); Assert.assertEquals(url1, url1Check); - Assert.assertEquals("Incorrect URL: " + url1, "solr://bob/db/5432/1.gz", url1); } @@ -164,6 +161,7 @@ public class SolrContentUrlBuilderTest { // Expected } + try { SolrContentUrlBuilder @@ -176,6 +174,7 @@ public class SolrContentUrlBuilderTest { // Expected } + try { SolrContentUrlBuilder @@ -188,6 +187,7 @@ public class SolrContentUrlBuilderTest { // Expected } + try { SolrContentUrlBuilder @@ -200,6 +200,7 @@ public class SolrContentUrlBuilderTest { // Expected } + try { SolrContentUrlBuilder diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentWriterTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentWriterTest.java index da58426bb..293e390fb 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentWriterTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/content/SolrContentWriterTest.java @@ -1,6 +1,5 @@ package org.alfresco.solr.content; -import org.alfresco.repo.content.ContentContext; import org.alfresco.service.cmr.repository.ContentReader; import org.alfresco.service.cmr.repository.ContentWriter; import org.apache.commons.io.FileUtils; @@ -13,12 +12,10 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -public class SolrContentWriterTest { - - +public class SolrContentWriterTest +{ private String solrHome = new File("./target/contentwriter/").getAbsolutePath(); - @After public void tearDown() throws IOException { @@ -26,37 +23,20 @@ public class SolrContentWriterTest { FileUtils.deleteDirectory(rootDir); } - - /** - * Generates a content context using a string. The URL part will be the same if the - * data provided is the same. - */ - private ContentContext createContentContext(String data) - { - return SolrContentUrlBuilder.start().add("data", data).getContentContext(); - } - - ContentWriter getContentWriter(String name) + private ContentWriter getContentWriter(String name) { return new SolrFileContentWriter(new File(solrHome, name), solrHome + "/" + name); } - - - ContentReader getContentReader(String name) + private ContentReader getContentReader(String name) { - return new SolrFileContentReader(new File(solrHome, name), solrHome + "/" + name); } - @Test public void contentByString() { - String filename = "abc"; - SolrContentStore store = new SolrContentStore(solrHome); - ContentWriter writer = getContentWriter(filename); File file = new File(solrHome, filename); @@ -84,7 +64,7 @@ public class SolrContentWriterTest { } @Test - public void contentByStream() throws Exception + public void contentByStream() { String filename = "cbs"; @@ -103,6 +83,4 @@ public class SolrContentWriterTest { Assert.assertEquals(bytes[1], bos.toByteArray()[1]); Assert.assertEquals(bytes[2], bos.toByteArray()[2]); } - - -} +} \ No newline at end of file From 5a0e9a66eeece388d26b5adeb0990cd89900c8ea Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 1 Nov 2019 11:18:14 +0000 Subject: [PATCH 48/76] SEARCH-1763 Ensure we run renamed integration tests during integration-test phase. --- pom.xml | 13 +++++++++++++ search-services/packaging/pom.xml | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index 5731b13fe..2cc03f504 100644 --- a/pom.xml +++ b/pom.xml @@ -54,6 +54,19 @@ true + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.2 + + + + integration-test + verify + + + + diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index 3417b6e2e..8c17b9e40 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -40,19 +40,6 @@ alfresco-search-services-${project.version} - - org.apache.maven.plugins - maven-failsafe-plugin - 2.22.2 - - - - integration-test - verify - - - - org.codehaus.mojo properties-maven-plugin From 34324db13e7fc6a10ae5522b71916deb78a1749f Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 1 Nov 2019 11:38:44 +0000 Subject: [PATCH 49/76] SEARCH-1763 Remove over a megabyte of output from the integration tests. --- .../src/test/java/org/alfresco/solr/query/AuthQueryIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthQueryIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthQueryIT.java index 55a93e309..82d9d70a6 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthQueryIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AuthQueryIT.java @@ -109,7 +109,6 @@ public class AuthQueryIT extends AuthDataLoad searchParameters.setQuery(queryString); Query query = dataModel.getFTSQuery(new Pair(searchParameters, Boolean.FALSE), solrQueryRequest, FTSQueryParser.RerankPhase.SINGLE_PASS); - System.out.println("##################### Query:"+query); TopDocs docs = solrIndexSearcher.search(query, count * 2 + 10); Assert.assertEquals(count, docs.totalHits); From 7821da2d21479240cad0754d7feea5d685f17ad0 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 1 Nov 2019 11:48:55 +0000 Subject: [PATCH 50/76] SEARCH-1763 Remove a load more output from the integration tests. --- .../java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java index 14a9a805d..33a0761e4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java @@ -760,8 +760,6 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer params.remove("distrib"); setDistributedParams(params); QueryResponse rsp = queryRandomShard(json, params); - System.out.println("Cluster Response:"+rsp); - System.out.println("Control Response:"+controlRsp); solrComparator.compareResponses(rsp, controlRsp); return rsp; } From 7e9d53a55ca85e05ab88137b523d6f72ce9dfb17 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 1 Nov 2019 13:35:57 +0000 Subject: [PATCH 51/76] SEARCH-1763 Convert AlfrescoCMISQParserPluginTest to an integration test. --- ...SQParserPluginTest.java => AlfrescoCMISQParserPluginIT.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/{AlfrescoCMISQParserPluginTest.java => AlfrescoCMISQParserPluginIT.java} (99%) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/AlfrescoCMISQParserPluginTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/AlfrescoCMISQParserPluginIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/AlfrescoCMISQParserPluginTest.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/AlfrescoCMISQParserPluginIT.java index b98fe72b0..e5803fe88 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/AlfrescoCMISQParserPluginTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/cmis/AlfrescoCMISQParserPluginIT.java @@ -27,7 +27,7 @@ import org.junit.Test; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class AlfrescoCMISQParserPluginTest extends LoadCMISData implements QueryConstants +public class AlfrescoCMISQParserPluginIT extends LoadCMISData implements QueryConstants { @Test public void cmisBasic() throws Exception From c67371fe754898dea7506684456f1c38440bd74c Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 1 Nov 2019 13:51:30 +0000 Subject: [PATCH 52/76] SEARCH-1914 Remove unused failing test. --- .../highlight/PostingsSolrHighlighterIT.java | 174 ------------------ 1 file changed, 174 deletions(-) delete mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/PostingsSolrHighlighterIT.java diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/PostingsSolrHighlighterIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/PostingsSolrHighlighterIT.java deleted file mode 100644 index 2b54f3af2..000000000 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/PostingsSolrHighlighterIT.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.alfresco.solr.highlight; - -import org.alfresco.solr.AbstractAlfrescoSolrIT; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.handler.component.AlfrescoSolrHighlighter; -import org.apache.solr.handler.component.HighlightComponent; -import org.apache.solr.highlight.SolrHighlighter; -import org.apache.solr.schema.IndexSchema; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import static junit.framework.TestCase.*; - -@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class PostingsSolrHighlighterIT extends AbstractAlfrescoSolrIT -{ - @BeforeClass - public static void beforeClass() throws Exception - { - initAlfrescoCore("schema.xml"); - // test our config is sane, just to be sure: - - // postingshighlighter should be used - SolrHighlighter highlighter = HighlightComponent.getHighlighter(getCore()); - assertTrue("wrong highlighter: " + highlighter.getClass(), highlighter instanceof AlfrescoSolrHighlighter); - - // 'text' and 'text3' should have offsets, 'text2' should not - IndexSchema schema = getCore().getLatestSchema(); - assertTrue(schema.getField("text").storeOffsetsWithPositions()); - assertTrue(schema.getField("text3").storeOffsetsWithPositions()); - assertFalse(schema.getField("text2").storeOffsetsWithPositions()); - } - - @Before - public void setUp() throws Exception - { - - // if you override setUp or tearDown, you better call - // the super classes version - clearIndex(); - assertU(adoc("text", "document one", "text2", "document one", "text3", "crappy document", "id", "101")); - assertU(adoc("text", "second document", "text2", "second document", "text3", "crappier document", "id", "102")); - assertU(commit()); - } - - @Test - public void testSimple() { - assertQ("simplest test", - req("q", "text:document", "sort", "id asc", "hl", "true"), - "count(//lst[@name='highlighting']/*)=2", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='document one'", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second document'"); - } - - public void testPagination() { - assertQ("pagination test", - req("q", "text:document", "sort", "id asc", "hl", "true", "rows", "1", "start", "1"), - "count(//lst[@name='highlighting']/*)=1", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second document'"); - } - - public void testEmptySnippet() { - assertQ("null snippet test", - req("q", "text:one OR *:*", "sort", "id asc", "hl", "true"), - "count(//lst[@name='highlighting']/*)=2", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='document one'", - "count(//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/*)=0"); - } - - public void testDefaultSummary() { - assertQ("null snippet test", - req("q", "text:one OR *:*", "sort", "id asc", "hl", "true", "hl.defaultSummary", "true"), - "count(//lst[@name='highlighting']/*)=2", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='document one'", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second document'"); - } - - public void testDifferentField() { - assertQ("highlighting text3", - req("q", "text3:document", "sort", "id asc", "hl", "true", "hl.fl", "text3"), - "count(//lst[@name='highlighting']/*)=2", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text3']/str='crappy document'", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text3']/str='crappier document'"); - } - - public void testTwoFields() { - assertQ("highlighting text and text3", - req("q", "text:document text3:document", "sort", "id asc", "hl", "true", "hl.fl", "text,text3"), - "count(//lst[@name='highlighting']/*)=2", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='document one'", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text3']/str='crappy document'", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second document'", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text3']/str='crappier document'"); - } - /** - public void testMisconfiguredField() { - ignoreException("was indexed without offsets"); - try { - assertQ("should fail, has no offsets", - req("q", "text2:document", "sort", "id asc", "hl", "true", "hl.fl", "text2")); - fail(); - } catch (Exception expected) { - // expected - } - resetExceptionIgnores(); - } - **/ - public void testTags() { - assertQ("different pre/post tags", - req("q", "text:document", "sort", "id asc", "hl", "true", "hl.tag.pre", "[", "hl.tag.post", "]"), - "count(//lst[@name='highlighting']/*)=2", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='[document] one'", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second [document]'"); - } - - public void testTagsPerField() { - assertQ("highlighting text and text3", - req("q", "text:document text3:document", "sort", "id asc", "hl", "true", "hl.fl", "text,text3", "f.text3.hl.tag.pre", "[", "f.text3.hl.tag.post", "]"), - "count(//lst[@name='highlighting']/*)=2", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='document one'", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text3']/str='crappy [document]'", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second document'", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text3']/str='crappier [document]'"); - } - - public void testBreakIterator() { - assertQ("different breakiterator", - req("q", "text:document", "sort", "id asc", "hl", "true", "hl.bs.type", "WORD"), - "count(//lst[@name='highlighting']/*)=2", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='document'", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='document'"); - } - - public void testBreakIterator2() { - assertU(adoc("text", "Document one has a first sentence. Document two has a second sentence.", "id", "103")); - assertU(commit()); - assertQ("different breakiterator", - req("q", "text:document", "sort", "id asc", "hl", "true", "hl.bs.type", "WHOLE"), - "//lst[@name='highlighting']/lst[@name='103']/arr[@name='text']/str='Document one has a first sentence. Document two has a second sentence.'"); - } - - public void testEncoder() { - assertU(adoc("text", "Document one has a first sentence.", "id", "103")); - assertU(commit()); - assertQ("html escaped", - req("q", "text:document", "sort", "id asc", "hl", "true", "hl.encoder", "html"), - "//lst[@name='highlighting']/lst[@name='103']/arr[@name='text']/str='Document one has a first <i>sentence</i>.'"); - } - - public void testWildcard() { - assertQ("simplest test", - req("q", "text:doc*ment", "sort", "id asc", "hl", "true", "hl.highlightMultiTerm", "true"), - "count(//lst[@name='highlighting']/*)=2", - "//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/str='document one'", - "//lst[@name='highlighting']/lst[@name='102']/arr[@name='text']/str='second document'"); - } -} From 8e0b555199b5a3c0ff1f9e8f7ff46ec9c78ada29 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 1 Nov 2019 13:56:39 +0000 Subject: [PATCH 53/76] SEARCH-1763 Fix server name in AdminHandlerDistributedIT. --- .../test/java/org/alfresco/solr/AdminHandlerDistributedIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java index 1f6255b13..2a9fe6aca 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java @@ -52,7 +52,7 @@ public class AdminHandlerDistributedIT extends AbstractAlfrescoDistributedIT @BeforeClass private static void initData() throws Throwable { - initSolrServers(2, "AdminHandlerDistributedTest", null); + initSolrServers(2, "AdminHandlerDistributedIT", null); } @AfterClass From 94ffe618b6bd33c2b3e175a66fb5663cbef369a4 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 1 Nov 2019 17:26:47 +0100 Subject: [PATCH 54/76] [contentStoreReplication] - code refactoring. - added documentation. - removed warnings. --- .../solr/handler/AlfrescoIndexFetcher.java | 107 ++++++++++++------ .../handler/AlfrescoReplicationHandler.java | 9 +- .../solr/handler/OldBackupDirectory.java | 4 +- .../alfresco/solr/handler/SnapShooter.java | 18 +-- 4 files changed, 91 insertions(+), 47 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index b9adc4298..ff93d0fbc 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -156,10 +156,12 @@ import java.util.zip.InflaterInputStream; *

    Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the * master.

    * + * This class has been modified in order to allow the alfresco contentstore to be efficiently replicated in a master slave environment. + * @author Elia */ class AlfrescoIndexFetcher { - public static final String REPLICATION_PROPERTIES = "replication.properties"; + static final String REPLICATION_PROPERTIES = "replication.properties"; static final String INDEX_REPLICATED_AT = "indexReplicatedAt"; static final String TIMES_INDEX_REPLICATED = "timesIndexReplicated"; static final String CONF_FILES_REPLICATED = "confFilesReplicated"; @@ -330,7 +332,7 @@ class AlfrescoIndexFetcher // we have checksums to compare - if (indexFileLen == backupIndexFileLen && indexFileChecksum == backupIndexFileChecksum) + if (indexFileLen == backupIndexFileLen && backupIndexFileChecksum!= null && indexFileChecksum == backupIndexFileChecksum) { compareResult.equal = true; return compareResult; @@ -377,7 +379,7 @@ class AlfrescoIndexFetcher } } - static boolean delTree(File dir) + private static boolean delTree(File dir) { try { @@ -402,7 +404,6 @@ class AlfrescoIndexFetcher params.set(CommonParams.QT, AlfrescoReplicationHandler.PATH); QueryRequest req = new QueryRequest(params); - // TODO modify to use shardhandler try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) { client.setSoTimeout(60000); @@ -418,6 +419,11 @@ class AlfrescoIndexFetcher /** * Fetches the list of files in a given index commit point and updates internal list of files to download. + * The list is composed by: + * - Index file list + * - Conf file list + * - TLog file list + * - Content store file list */ @SuppressWarnings("unchecked") private void fetchFileList(long indexGeneration, long contentStoreGeneration) throws IOException @@ -469,7 +475,7 @@ class AlfrescoIndexFetcher List> infomap = contentStoreMap.get(SolrContentStore.INFO); fullContentStoreReplication = of(infomap).stream().flatMap(List::stream) .map(e -> e.get(SolrContentStore.FULL_REPLICATION)) - .map(e -> Boolean.class.isInstance(e) ? (Boolean) e : false).findFirst().orElse(false); + .map(e -> e instanceof Boolean ? (Boolean) e : false).findFirst().orElse(false); } @@ -492,6 +498,7 @@ class AlfrescoIndexFetcher * * @param forceReplication force a replication in all cases * @param forceCoreReload force a core reload in all cases + * @param replicateContentStore downloads the changed content store files * @return true on success, false if slave is already in sync * @throws IOException if an exception occurs */ @@ -763,6 +770,7 @@ class AlfrescoIndexFetcher if (tlogFilesToDownload != null) { + assert tmpTlogDir != null; bytesDownloaded += downloadTlogFiles(tmpTlogDir, latestGeneration); reloadCore = true; // reload update log } @@ -783,19 +791,17 @@ class AlfrescoIndexFetcher if (fullContentStoreReplication) { - deleteUnnecessaryContentStoreFiles(contentStore.getRootLocation()); + cleanUpContentStore(contentStore.getRootLocation()); } contentStore.setLastCommittedVersion(masterContentStoreVersion); } final long timeTakenSeconds = getReplicationTimeElapsed(); - final Long bytesDownloadedPerSecond = (timeTakenSeconds != 0 ? - new Long(bytesDownloaded / timeTakenSeconds) : + final Long bytesDownloadedPerSecond = (timeTakenSeconds != 0 ? bytesDownloaded / timeTakenSeconds : null); LOG.info("Total time taken for download (fullCopy={},bytesDownloaded={}) : {} secs ({} bytes/sec)", - new Object[] { isFullCopyNeeded, bytesDownloaded, timeTakenSeconds, - bytesDownloadedPerSecond }); + isFullCopyNeeded, bytesDownloaded, timeTakenSeconds, bytesDownloadedPerSecond); if (indexReplicationNeeded) { @@ -809,7 +815,9 @@ class AlfrescoIndexFetcher { successfulInstall = solrCore.modifyIndexProps(tmpIdxDirName); if (successfulInstall) + { deleteTmpIdxDir = false; + } } else { @@ -847,7 +855,9 @@ class AlfrescoIndexFetcher { successfulInstall = solrCore.modifyIndexProps(tmpIdxDirName); if (successfulInstall) + { deleteTmpIdxDir = false; + } } else { @@ -992,7 +1002,12 @@ class AlfrescoIndexFetcher return bytesDownloaded; } - private void cleanup(final SolrCore core, Directory tmpIndexDir, Directory indexDir, boolean deleteTmpIdxDir, File tmpTlogDir, boolean successfulInstall) + private void cleanup(final SolrCore core, + Directory tmpIndexDir, + Directory indexDir, + boolean deleteTmpIdxDir, + File tmpTlogDir, + boolean successfulInstall) { try { @@ -1263,9 +1278,7 @@ class AlfrescoIndexFetcher RefCounted searcher = null; IndexCommit commitPoint; // must get the latest solrCore object because the one we have might be closed because of a reload - // todo stop keeping solrCore around - SolrCore core = solrCore.getCoreContainer().getCore(solrCore.getName()); - try + try (SolrCore core = solrCore.getCoreContainer().getCore(solrCore.getName())) { Future[] waitSearcher = new Future[1]; searcher = core.getSearcher(true, true, waitSearcher, true); @@ -1288,7 +1301,6 @@ class AlfrescoIndexFetcher { searcher.decref(); } - core.close(); } // update the commit point in replication handler @@ -1601,15 +1613,19 @@ class AlfrescoIndexFetcher private List makeTmpConfDirFileList(File dir, List fileList) { File[] files = dir.listFiles(); - for (File file : files) + + if (files != null) { - if (file.isFile()) + for (File file : files) { - fileList.add(file); - } - else if (file.isDirectory()) - { - fileList = makeTmpConfDirFileList(file, fileList); + if (file.isFile()) + { + fileList.add(file); + } + else if (file.isDirectory()) + { + fileList = makeTmpConfDirFileList(file, fileList); + } } } return fileList; @@ -1661,6 +1677,12 @@ class AlfrescoIndexFetcher } } + /** + * Copy the downloaded content store files into the contentstore directory + * @param tmpContentStoreDir + * @param contentStorePath + * @throws IOException + */ private void copyTmpContentStoreToContentStore(File tmpContentStoreDir, String contentStorePath) throws IOException { @@ -1692,6 +1714,11 @@ class AlfrescoIndexFetcher } } + /** + * Deletes the files in filesToDelete list from contentStore + * @param contentStorePath + * @param filesToDelete + */ private void deleteContentStoreFiles(String contentStorePath, List> filesToDelete) { filesToDelete.stream().map(f -> (String) f.get(NAME)).forEach(p -> { @@ -1702,9 +1729,12 @@ class AlfrescoIndexFetcher LOG.info("deleted {} files from content store", filesToDelete.size()); } - private void deleteUnnecessaryContentStoreFiles(String contentStorePath) + /** + * Deletes from contentstore all the files that has not been updated. + * @param contentStorePath + */ + private void cleanUpContentStore(String contentStorePath) { - AtomicInteger fileDeleted = new AtomicInteger(); Set fileNames = contentStoreFilesToDownload.stream().map(e -> (String) e.get(NAME)) .collect(Collectors.toSet()); @@ -1949,7 +1979,7 @@ class AlfrescoIndexFetcher } } - public void destroy() + void destroy() { abortFetch(); } @@ -2076,14 +2106,12 @@ class AlfrescoIndexFetcher { FileChannel fileChannel; File file; - private File copy2Dir; private FileOutputStream fileOutputStream; LocalFsFile(File dir, String saveAs) throws IOException { - this.copy2Dir = dir; - this.file = new File(copy2Dir, saveAs); + this.file = new File(dir, saveAs); File parentDir = this.file.getParentFile(); if (!parentDir.exists()) @@ -2281,6 +2309,7 @@ class AlfrescoIndexFetcher //compare the checksum as sent from the master if (includeChecksum) { + assert checksum != null; checksum.reset(); checksum.update(buf, 0, packetSize); long checkSumClient = checksum.getValue(); @@ -2436,6 +2465,9 @@ class AlfrescoIndexFetcher } } + /** + * File fetcher specialized for downloading index files + */ private class DirectoryFileFetcher extends FileFetcher { DirectoryFileFetcher(Directory tmpIndexDir, Map fileDetails, String saveAs, String solrParamOutput, long latestGen) @@ -2445,6 +2477,10 @@ class AlfrescoIndexFetcher } } + + /** + * File fetcher specialized for downloading conf and tlogs files + */ class LocalFsFileFetcher extends FileFetcher { LocalFsFileFetcher(File dir, Map fileDetails, String saveAs, String solrParamOutput, long latestGen) @@ -2454,13 +2490,18 @@ class AlfrescoIndexFetcher } } + + /** + * File fetcher specialized for downloading conf and contentstore files. + * In order to handle the possibly high number of files, the requested files are downloaded in a single stream. + */ class ContentStoreFetcher extends FileFetcher { private final File dir; private final Set filesToDownload; private final Set filesDownloaded; - ContentStoreFetcher(File dir, String solrParamOutput, List> filesDetails) throws IOException + ContentStoreFetcher(File dir, String solrParamOutput, List> filesDetails) { super(solrParamOutput, 0); this.dir = dir; @@ -2494,7 +2535,7 @@ class AlfrescoIndexFetcher } } - public void fetchContentStore() throws Exception + void fetchContentStore() throws Exception { this.fetchFile(); } @@ -2509,7 +2550,6 @@ class AlfrescoIndexFetcher params.set(GENERATION, Long.toString(indexGen)); params.set(CommonParams.QT, AlfrescoReplicationHandler.PATH); - List l = new ArrayList<>(); params.set(CONTENT_STORE_FILE_LIST, filesToDownload.toArray(String[]::new)); //add the version to download. This is used to reserve the download @@ -2637,6 +2677,7 @@ class AlfrescoIndexFetcher //compare the checksum as sent from the master if (includeChecksum) { + assert checksum != null; checksum.reset(); checksum.update(buf, 0, packetSize); long checkSumClient = checksum.getValue(); @@ -2684,8 +2725,7 @@ class AlfrescoIndexFetcher } catch (Exception e) { - LOG.warn("Error in fetching file: {} (downloaded {} of {} bytes)", - new Object[] { fileName, bytesDownloaded, size, e }); + LOG.warn("Error in fetching file: {} (downloaded {} of {} bytes)", fileName, bytesDownloaded, size, e); //for any failure, increment the error count errorCount++; //if it fails for the same packet for MAX_RETRIES fail and come out @@ -2699,6 +2739,5 @@ class AlfrescoIndexFetcher return ERR; } } - } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java index 6400050d4..be01b48b4 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java @@ -144,6 +144,11 @@ import static org.apache.solr.common.params.CommonParams.NAME; * or command=disablepoll) * * @since solr 1.4 + * + * + * This class has been modified in order to allow the alfresco contentstore to be efficiently replicated in a master slave environment. + * @author Elia + * */ public class AlfrescoReplicationHandler extends RequestHandlerBase implements SolrCoreAware { @@ -496,7 +501,7 @@ public class AlfrescoReplicationHandler extends RequestHandlerBase implements So private volatile AlfrescoIndexFetcher currentAlfrescoIndexFetcher; - boolean acquireContentStoreReplicationTask() + private boolean acquireContentStoreReplicationTask() { contentStoreReplicationLock.lock(); if (!isContentStoreReplicating) @@ -1361,7 +1366,7 @@ public class AlfrescoReplicationHandler extends RequestHandlerBase implements So float totalPercent = 0; long downloadSpeed = 0; if (bytesToDownload > 0) - totalPercent = (bytesDownloaded * 100) / bytesToDownload; + totalPercent = (bytesDownloaded * 100.f) / bytesToDownload; if (elapsed > 0) downloadSpeed = (bytesDownloaded / elapsed); if (currFile != null) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java index 30db17bf8..3c188ea2b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java @@ -56,7 +56,7 @@ class OldBackupDirectory implements Comparable private String dirName; private Optional timestamp = Optional.empty(); - public OldBackupDirectory(URI basePath, String dirName) + OldBackupDirectory(URI basePath, String dirName) { this.dirName = Objects.requireNonNull(dirName); this.basePath = Objects.requireNonNull(basePath); @@ -79,7 +79,7 @@ class OldBackupDirectory implements Comparable return this.basePath.resolve(dirName); } - public String getDirName() + String getDirName() { return dirName; } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java index 99652cc71..680d915f3 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java @@ -87,7 +87,7 @@ public class SnapShooter @Deprecated public SnapShooter(SolrCore core, String location, String snapshotName) { - String snapDirStr = null; + String snapDirStr; // Note - This logic is only applicable to the usecase where a shared file-system is exposed via // local file-system interface (primarily for backwards compatibility). For other use-cases, users // will be required to specify "location" where the backup should be stored. @@ -102,7 +102,7 @@ public class SnapShooter initialize(new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null); } - public SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) + SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) { initialize(backupRepo, core, location, snapshotName, commitName); } @@ -140,7 +140,7 @@ public class SnapShooter return this.baseSnapDirPath; } - public void validateDeleteSnapshot() + void validateDeleteSnapshot() { Objects.requireNonNull(this.snapshotName); @@ -158,7 +158,7 @@ public class SnapShooter break; } } - if (dirFound == false) + if (!dirFound) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Snapshot " + snapshotName + " cannot be found in directory: " + baseSnapDirPath); @@ -171,12 +171,12 @@ public class SnapShooter } } - protected void deleteSnapAsync(final AlfrescoReplicationHandler alfrescoReplicationHandler) + void deleteSnapAsync(final AlfrescoReplicationHandler alfrescoReplicationHandler) { new Thread(() -> deleteNamedSnapshot(alfrescoReplicationHandler)).start(); } - public void validateCreateSnapshot() throws IOException + void validateCreateSnapshot() throws IOException { // Note - Removed the current behavior of creating the directory hierarchy. // Do we really need to provide this support? @@ -231,7 +231,7 @@ public class SnapShooter } } - public void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, Consumer result) + void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, Consumer result) { solrCore.getDeletionPolicy().saveCommitPoint(indexCommit.getGeneration()); @@ -267,7 +267,7 @@ public class SnapShooter } // note: remember to reserve the indexCommit first so it won't get deleted concurrently - protected NamedList createSnapshot(final IndexCommit indexCommit) throws Exception + private NamedList createSnapshot(final IndexCommit indexCommit) throws Exception { LOG.info("Creating backup snapshot " + (snapshotName == null ? "" : snapshotName) + " at " + baseSnapDirPath); @@ -372,6 +372,6 @@ public class SnapShooter alfrescoReplicationHandler.snapShootDetails = details; } - public static final String DATE_FMT = "yyyyMMddHHmmssSSS"; + private static final String DATE_FMT = "yyyyMMddHHmmssSSS"; } From a9c2c046538dbbb53a28a6795db366c6000ca559 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Mon, 4 Nov 2019 11:29:17 +0100 Subject: [PATCH 55/76] [contentStoreReplication] added notation suppressSSL in IT. --- .../handler/ContentStoreReplicationIT.java | 268 ++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java new file mode 100644 index 000000000..a1a7fb206 --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java @@ -0,0 +1,268 @@ +/* + * Copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.handler; + +import java.io.IOException; +import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.client.Acl; +import org.alfresco.solr.client.AclChangeSet; +import org.alfresco.solr.client.AclReaders; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.Transaction; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Properties; + +import static java.util.Collections.singletonList; +import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; +import static org.alfresco.solr.AlfrescoSolrUtils.getNode; +import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; +import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; +import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; +import static org.alfresco.solr.utils.AlfrescoFileUtils.areDirectoryEquals; +import static org.carrot2.shaded.guava.common.collect.ImmutableList.of; + +/** + * @author Elia Porciani + * + * This test check if the synchronization of contentstore between master and slave is done correctly. + */ +@SolrTestCaseJ4.SuppressSSL +public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedTest { + + protected static JettySolrRunner master; + protected static JettySolrRunner slave; + protected static SolrClient masterClient; + protected static SolrClient slaveClient; + + protected static Path masterSolrHome; + protected static Path slaveSolrHome; + + protected static Path masterContentStore; + protected static Path slaveContentStore; + + private static Acl acl; + private static final int MILLIS_TIMOUT = 80000; + + @BeforeClass + public static void createMasterSlaveEnv() throws Exception + { + Properties properties = new Properties(); + + clientShards = new ArrayList<>(); + solrShards = new ArrayList<>(); + solrCollectionNameToStandaloneClient = new HashMap<>(); + jettyContainers = new HashMap<>(); + + String coreName = "master"; + boolean basicAuth = Boolean.parseBoolean(properties.getProperty("BasicAuth", "false")); + + String masterKey = "master/solrHome"; + String slaveKey = "slave/solrHome"; + + master = createJetty(masterKey, basicAuth); + addCoreToJetty(masterKey, coreName, coreName, null); + startJetty(master); + + + String slaveCoreName = "slave"; + slave = createJetty(slaveKey, basicAuth); + addCoreToJetty(slaveKey, slaveCoreName, slaveCoreName, null); + setMasterUrl(slaveKey, slaveCoreName, master.getBaseUrl().toString() + "/master"); + + startJetty(slave); + + String masterStr = buildUrl(master.getLocalPort()) + "/" + coreName; + String slaveStr = buildUrl(slave.getLocalPort()) + "/" + slaveCoreName; + + masterClient = createNewSolrClient(masterStr); + slaveClient = createNewSolrClient(slaveStr); + + masterSolrHome = testDir.toPath().resolve(masterKey); + slaveSolrHome = testDir.toPath().resolve(slaveKey); + masterContentStore = testDir.toPath().resolve("master/contentstore"); + slaveContentStore = testDir.toPath().resolve("slave/contentstore"); + + AclChangeSet aclChangeSet = getAclChangeSet(1); + + acl = getAcl(aclChangeSet); + AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + + indexAclChangeSet(aclChangeSet, + of(acl), + of(aclReaders)); + } + + + @AfterClass + public static void cleanupMasterSlave() throws Exception { + master.stop(); + slave.stop(); + FileUtils.forceDelete(new File(masterSolrHome.getParent().toUri())); + FileUtils.forceDelete(new File(slaveSolrHome.getParent().toUri())); + } + + + @Test + public void contentStoreReplicationTest() throws Exception { + // ADD 250 nodes and check they are replicated + int numNodes = 250; + Transaction bigTxn = getTransaction(0, numNodes); + List nodes = new ArrayList<>(); + List nodeMetaDatas = new ArrayList<>(); + for(int i = 0; i updateNodes = new ArrayList<>(); + List updateNodeMetaDatas = new ArrayList<>(); + for(int i = numNodes; i < totalNodes; i++) { + Node node = getNode(i, updateTx, acl, Node.SolrApiNodeStatus.UPDATED); + updateNodes.add(node); + NodeMetaData nodeMetaData = getNodeMetaData(node, updateTx, acl, "mike", null, false); + node.setNodeRef(nodeMetaData.getNodeRef().toString()); + updateNodeMetaDatas.add(nodeMetaData); + } + + indexTransaction(updateTx, updateNodes, updateNodeMetaDatas); + + waitForDocCountCore(masterClient, + luceneToSolrQuery(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world"))), + numNodes + numUpdates, MILLIS_TIMOUT, System.currentTimeMillis()); + + filesInMasterContentStore = Files.walk(Paths.get(masterContentStore.toUri().resolve("_DEFAULT_"))) + .filter(Files::isRegularFile) + .count(); + + Assert.assertEquals( "master contentStore should have " + totalNodes + "files", totalNodes, filesInMasterContentStore); + assertTrue("slave content store is not in sync after timeout", waitForContentStoreSync(MILLIS_TIMOUT)); + + + // DELETES 30 nodes + int numDeletes = 30; + Transaction deleteTx = getTransaction(numDeletes, 0); + + totalNodes = numNodes + numUpdates - numDeletes; + List deleteNodes = new ArrayList<>(); + List deleteNodeMetaDatas = new ArrayList<>(); + + for(int i = 0; i Date: Mon, 4 Nov 2019 11:30:19 +0100 Subject: [PATCH 56/76] [contentStoreReplication] Modified template with new replicationHandler class name --- .../solr/instance/templates/rerank/conf/solrconfig.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml index f41aac283..ac76d0d7b 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml @@ -1154,7 +1154,8 @@ https://wiki.apache.org/solr/SolrCloud/ --> - + + diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index 3417b6e2e..ed5fca082 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -163,13 +163,13 @@ - - - - - - - + + + + + + + diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index 6faa2cd2d..c29663f5e 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -74,6 +74,19 @@ guessencoding-1.4.jar http://docs.codehaus.org/display/GUESSENC/ xml-apis-1.4.01.jar https://github.com/FasterXML/jackson jug-2.0.0-asl.jar http://jug.safehaus.org/ acegi-security-0.8.2_patched.jar http://sourceforge.net/projects/acegisecurity/ +jetty-continuation-9.3.27.v20190418.jar +jetty-deploy-9.3.27.v20190418.jar +jetty-http-9.3.27.v20190418.jar +jetty-io-9.3.27.v20190418.jar +jetty-jmx-9.3.27.v20190418.jar +jetty-rewrite-9.3.27.v20190418.jar +jetty-security-9.3.27.v20190418.jar +jetty-server-9.3.27.v20190418.jar +jetty-servlet-9.3.27.v20190418.jar +jetty-servlets-9.3.27.v20190418.jar +jetty-util-9.3.27.v20190418.jar +jetty-webapp-9.3.27.v20190418.jar +jetty-xml-9.3.27.v20190418.jar === CDDL 1.0 === @@ -148,19 +161,6 @@ javax.servlet-api-3.1.0.jar jcl-over-slf4j-1.7.7.jar jdom-1.0.jar jempbox-1.8.13.jar -jetty-continuation-9.3.14.v20161028.jar -jetty-deploy-9.3.14.v20161028.jar -jetty-http-9.3.14.v20161028.jar -jetty-io-9.3.14.v20161028.jar -jetty-jmx-9.3.14.v20161028.jar -jetty-rewrite-9.3.14.v20161028.jar -jetty-security-9.3.14.v20161028.jar -jetty-server-9.3.14.v20161028.jar -jetty-servlet-9.3.14.v20161028.jar -jetty-servlets-9.3.14.v20161028.jar -jetty-util-9.3.14.v20161028.jar -jetty-webapp-9.3.14.v20161028.jar -jetty-xml-9.3.14.v20161028.jar jmatio-1.2.jar joda-time-2.2.jar jsonic-1.2.7.jar @@ -168,28 +168,28 @@ jul-to-slf4j-1.7.7.jar juniversalchardet-1.0.3.jar langdetect-1.1-20120112.jar log4j-1.2.17.jar -lucene-analyzers-common-6.6.5-patched.jar -lucene-analyzers-icu-6.6.5-patched.jar -lucene-analyzers-kuromoji-6.6.5-patched.jar -lucene-analyzers-morfologik-6.6.5-patched.jar -lucene-analyzers-phonetic-6.6.5-patched.jar -lucene-analyzers-smartcn-6.6.5-patched.jar -lucene-analyzers-stempel-6.6.5-patched.jar -lucene-backward-codecs-6.6.5-patched.jar -lucene-classification-6.6.5-patched.jar -lucene-codecs-6.6.5-patched.jar -lucene-core-6.6.5-patched.jar -lucene-expressions-6.6.5-patched.jar -lucene-grouping-6.6.5-patched.jar -lucene-highlighter-6.6.5-patched.jar -lucene-join-6.6.5-patched.jar -lucene-memory-6.6.5-patched.jar -lucene-misc-6.6.5-patched.jar -lucene-queries-6.6.5-patched.jar -lucene-queryparser-6.6.5-patched.jar -lucene-sandbox-6.6.5-patched.jar -lucene-spatial-extras-6.6.5-patched.jar -lucene-suggest-6.6.5-patched.jar +lucene-analyzers-common-6.6.5-patched.1.jar +lucene-analyzers-icu-6.6.5-patched.1.jar +lucene-analyzers-kuromoji-6.6.5-patched.1.jar +lucene-analyzers-morfologik-6.6.5-patched.1.jar +lucene-analyzers-phonetic-6.6.5-patched.1.jar +lucene-analyzers-smartcn-6.6.5-patched.1.jar +lucene-analyzers-stempel-6.6.5-patched.1.jar +lucene-backward-codecs-6.6.5-patched.1.jar +lucene-classification-6.6.5-patched.1.jar +lucene-codecs-6.6.5-patched.1.jar +lucene-core-6.6.5-patched.1.jar +lucene-expressions-6.6.5-patched.1.jar +lucene-grouping-6.6.5-patched.1.jar +lucene-highlighter-6.6.5-patched.1.jar +lucene-join-6.6.5-patched.1.jar +lucene-memory-6.6.5-patched.1.jar +lucene-misc-6.6.5-patched.1.jar +lucene-queries-6.6.5-patched.1.jar +lucene-queryparser-6.6.5-patched.1.jar +lucene-sandbox-6.6.5-patched.1.jar +lucene-spatial-extras-6.6.5-patched.1.jar +lucene-suggest-6.6.5-patched.1.jar metadata-extractor-2.9.1.jar metrics-core-3.2.2.jar metrics-ganglia-3.2.2.jar @@ -213,11 +213,11 @@ rome-1.5.1.jar simple-xml-2.7.1.jar slf4j-api-1.7.7.jar slf4j-log4j12-1.7.7.jar -solr-analysis-extras-6.6.5-patched.jar -solr-clustering-6.6.5-patched.jar -solr-core-6.6.5-patched.jar -solr-langid-6.6.5-patched.jar -solr-solrj-6.6.5-patched.jar +solr-analysis-extras-6.6.5-patched.1.jar +solr-clustering-6.6.5-patched.1.jar +solr-core-6.6.5-patched.1.jar +solr-langid-6.6.5-patched.1.jar +solr-solrj-6.6.5-patched.1.jar spatial4j-0.6.jar start.jar stax2-api-3.1.4.jar From c32ff1a922a878457abee6e66c20c294512e3390 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 4 Nov 2019 16:37:14 +0000 Subject: [PATCH 58/76] SEARCH-1906 Add license reference for Jetty. --- .../src/main/resources/licenses/notice.txt | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index c29663f5e..12e46175c 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -74,19 +74,19 @@ guessencoding-1.4.jar http://docs.codehaus.org/display/GUESSENC/ xml-apis-1.4.01.jar https://github.com/FasterXML/jackson jug-2.0.0-asl.jar http://jug.safehaus.org/ acegi-security-0.8.2_patched.jar http://sourceforge.net/projects/acegisecurity/ -jetty-continuation-9.3.27.v20190418.jar -jetty-deploy-9.3.27.v20190418.jar -jetty-http-9.3.27.v20190418.jar -jetty-io-9.3.27.v20190418.jar -jetty-jmx-9.3.27.v20190418.jar -jetty-rewrite-9.3.27.v20190418.jar -jetty-security-9.3.27.v20190418.jar -jetty-server-9.3.27.v20190418.jar -jetty-servlet-9.3.27.v20190418.jar -jetty-servlets-9.3.27.v20190418.jar -jetty-util-9.3.27.v20190418.jar -jetty-webapp-9.3.27.v20190418.jar -jetty-xml-9.3.27.v20190418.jar +jetty-continuation-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-deploy-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-http-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-io-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-jmx-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-rewrite-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-security-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-server-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-servlet-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-servlets-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-util-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-webapp-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +jetty-xml-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html === CDDL 1.0 === From f7c4e6d3239e4330fe9bccadc92f7c5c17e79b56 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 5 Nov 2019 14:50:16 +0100 Subject: [PATCH 59/76] [contentStoreReplication] set correct log level --- .../java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index ff93d0fbc..ee1666778 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -975,7 +975,7 @@ class AlfrescoIndexFetcher if (contentStoreFilesToDownloadFiltered.size() != contentStoreFilesToDownload.size()) { - LOG.warn("content store replication: some of the files are already in sync. {} files to download", + LOG.info("content store replication: some of the files are already in sync. {} files to download", contentStoreFilesToDownloadFiltered.size()); } From 9cf970f60a3943ed7280199fa38bdec66809c54e Mon Sep 17 00:00:00 2001 From: agazzarini Date: Tue, 5 Nov 2019 19:27:49 +0100 Subject: [PATCH 60/76] [ SEARCH-1917 ] First working draft --- .../solr/AlfrescoCoreAdminHandler.java | 786 +++++++++--------- .../alfresco/solr/HandlerReportBuilder.java | 502 ++++++----- .../solr/lifecycle/SolrCoreLoadListener.java | 6 +- ...Publisher.java => CoreStatePublisher.java} | 33 +- .../solr/tracker/MetadataTracker.java | 34 +- ...sher.java => SlaveCoreStatePublisher.java} | 14 +- 6 files changed, 741 insertions(+), 634 deletions(-) rename search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/{NodeStatePublisher.java => CoreStatePublisher.java} (91%) rename search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/{SlaveNodeStatePublisher.java => SlaveCoreStatePublisher.java} (90%) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index f714307dc..9badb9506 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -27,7 +27,8 @@ import static org.alfresco.solr.HandlerOfResources.getSafeLong; import static org.alfresco.solr.HandlerOfResources.openResource; import static org.alfresco.solr.HandlerOfResources.updatePropertiesFile; import static org.alfresco.solr.HandlerOfResources.updateSharedProperties; -import static org.alfresco.solr.HandlerReportBuilder.addCoreSummary; +import static org.alfresco.solr.HandlerReportBuilder.addMasterOrStandaloneCoreSummary; +import static org.alfresco.solr.HandlerReportBuilder.addSlaveCoreSummary; import static org.alfresco.solr.HandlerReportBuilder.buildAclReport; import static org.alfresco.solr.HandlerReportBuilder.buildAclTxReport; import static org.alfresco.solr.HandlerReportBuilder.buildNodeReport; @@ -36,7 +37,6 @@ import static org.alfresco.solr.HandlerReportBuilder.buildTxReport; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -66,6 +66,8 @@ import org.alfresco.solr.tracker.DBIDRangeRouter; import org.alfresco.solr.tracker.DocRouter; import org.alfresco.solr.tracker.IndexHealthReport; import org.alfresco.solr.tracker.MetadataTracker; +import org.alfresco.solr.tracker.CoreStatePublisher; +import org.alfresco.solr.tracker.SlaveCoreStatePublisher; import org.alfresco.solr.tracker.SolrTrackerScheduler; import org.alfresco.solr.tracker.Tracker; import org.alfresco.solr.tracker.TrackerRegistry; @@ -92,15 +94,15 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler protected static final Logger LOGGER = LoggerFactory.getLogger(AlfrescoCoreAdminHandler.class); private static final String ARG_ACLTXID = "acltxid"; - protected static final String ARG_TXID = "txid"; + static final String ARG_TXID = "txid"; private static final String ARG_ACLID = "aclid"; private static final String ARG_NODEID = "nodeid"; private static final String ARG_QUERY = "query"; - public static final String DATA_DIR_ROOT = "data.dir.root"; + private static final String DATA_DIR_ROOT = "data.dir.root"; public static final String ALFRESCO_DEFAULTS = "create.alfresco.defaults"; - public static final String NUM_SHARDS = "num.shards"; - public static final String SHARD_IDS = "shard.ids"; - public static final String DEFAULT_TEMPLATE = "rerank"; + private static final String NUM_SHARDS = "num.shards"; + private static final String SHARD_IDS = "shard.ids"; + static final String DEFAULT_TEMPLATE = "rerank"; static final String ALFRESCO_CORE_NAME = "alfresco"; static final String ARCHIVE_CORE_NAME = "archive"; @@ -130,31 +132,22 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler */ public void startup(CoreContainer coreContainer) { - LOGGER.info("Starting Alfresco core container services"); + LOGGER.info("Starting Alfresco Core Administration Services"); trackerRegistry = new TrackerRegistry(); informationServers = new ConcurrentHashMap<>(); this.scheduler = new SolrTrackerScheduler(this); String createDefaultCores = ConfigUtil.locateProperty(ALFRESCO_DEFAULTS, ""); - int numShards = Integer.valueOf(ConfigUtil.locateProperty(NUM_SHARDS, "1")); + int numShards = Integer.parseInt(ConfigUtil.locateProperty(NUM_SHARDS, "1")); String shardIds = ConfigUtil.locateProperty(SHARD_IDS, null); if (createDefaultCores != null && !createDefaultCores.isEmpty()) { - Runnable runnable = () -> + Thread thread = new Thread(() -> { - try - { - TimeUnit.SECONDS.sleep(10); //Wait a little for the container to start up - } - catch (InterruptedException e) - { - //Don't care - } + waitForTenSeconds(); setupNewDefaultCores(createDefaultCores, numShards, 1, 1, 1, shardIds); - }; - - Thread thread = new Thread(runnable); + }); thread.start(); } } @@ -179,7 +172,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * @param numNodes - Not sure why the core needs to know this. * @param shardIds A comma separated list of shard ids for this core (or null). */ - void setupNewDefaultCores(String names, int numShards, int replicationFactor, int nodeInstance, int numNodes, String shardIds) + private void setupNewDefaultCores(String names, int numShards, int replicationFactor, int nodeInstance, int numNodes, String shardIds) { try { @@ -195,7 +188,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .filter(coreName -> !coreName.isEmpty()) .forEach(coreName -> { LOGGER.info("Attempting to create default alfresco core: {}", coreName); - if (!STORE_REF_MAP.keySet().contains(coreName)) + if (!STORE_REF_MAP.containsKey(coreName)) { throw new AlfrescoRuntimeException("Invalid '" + ALFRESCO_DEFAULTS + "' permitted values are " + STORE_REF_MAP.keySet()); } @@ -257,7 +250,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } catch (ClassNotFoundException e) { - return; + // Do nothing here } catch (Exception e) { @@ -267,25 +260,25 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler protected void handleCustomAction(SolrQueryRequest req, SolrQueryResponse rsp) { - LOGGER.info("######## Handle Custom Action ###########"); SolrParams params = req.getParams(); String cname = params.get(CoreAdminParams.CORE); String action = params.get(CoreAdminParams.ACTION); action = action==null?"":action.toUpperCase(); try { - switch (action) { + switch (action) + { case "NEWCORE": newCore(req, rsp); break; case "UPDATECORE": - updateCore(req, rsp); + updateCore(req); break; case "UPDATESHARED": - updateShared(req, rsp); + updateShared(req); break; case "REMOVECORE": - removeCore(req, rsp); + removeCore(req); break; case "NEWDEFAULTINDEX": newDefaultCore(req, rsp); @@ -315,58 +308,82 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler actionREPORT(rsp, params, cname); break; case "PURGE": - if (cname != null) { + if (cname != null) + { actionPURGE(params, cname); - } else { - for (String coreName : getTrackerRegistry().getCoreNames()) { + } + else + { + for (String coreName : getTrackerRegistry().getCoreNames()) + { actionPURGE(params, coreName); } } break; case "REINDEX": - if (cname != null) { + if (cname != null) + { actionREINDEX(params, cname); - } else { - for (String coreName : getTrackerRegistry().getCoreNames()) { + } + else + { + for (String coreName : getTrackerRegistry().getCoreNames()) + { actionREINDEX(params, coreName); } } break; case "RETRY": - if (cname != null) { + if (cname != null) + { actionRETRY(rsp, cname); - } else { - for (String coreName : getTrackerRegistry().getCoreNames()) { + } + else + { + for (String coreName : getTrackerRegistry().getCoreNames()) + { actionRETRY(rsp, coreName); } } break; case "INDEX": - if (cname != null) { + if (cname != null) + { actionINDEX(params, cname); - } else { - for (String coreName : getTrackerRegistry().getCoreNames()) { + } + else + { + for (String coreName : getTrackerRegistry().getCoreNames()) + { actionINDEX(params, coreName); } } break; case "FIX": - if (cname != null) { + if (cname != null) + { actionFIX(cname); - } else { - for (String coreName : getTrackerRegistry().getCoreNames()) { + } + else + { + for (String coreName : getTrackerRegistry().getCoreNames()) + { actionFIX(coreName); } } break; case "SUMMARY": - if (cname != null) { - NamedList report = new SimpleOrderedMap(); + if (cname != null) + { + NamedList report = new SimpleOrderedMap<>(); actionSUMMARY(params, report, cname); rsp.add("Summary", report); - } else { - NamedList report = new SimpleOrderedMap(); - for (String coreName : getTrackerRegistry().getCoreNames()) { + } + else + { + NamedList report = new SimpleOrderedMap<>(); + for (String coreName : getTrackerRegistry().getCoreNames()) + { actionSUMMARY(params, report, coreName); } rsp.add("Summary", report); @@ -374,7 +391,8 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler break; case "LOG4J": String resource = "log4j-solr.properties"; - if (params.get("resource") != null) { + if (params.get("resource") != null) + { resource = params.get("resource"); } initResourceBasedLogging(resource); @@ -391,26 +409,23 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } - private boolean newCore(SolrQueryRequest req, SolrQueryResponse rsp) { + private boolean newCore(SolrQueryRequest req, SolrQueryResponse rsp) + { SolrParams params = req.getParams(); req.getContext(); - // If numCore > 1 we are createing a collection of cores for a sole node in a cluster + // If numCore > 1 we are creating a collection of cores for a sole node in a cluster int numShards = params.getInt("numShards", 1); - String store = ""; - if (params.get("storeRef") != null) { - store = params.get("storeRef"); - } - if ((store == null) || (store.length() == 0)) { + String store = params.get("storeRef"); + if (store == null || store.trim().length() == 0) + { return false; } + StoreRef storeRef = new StoreRef(store); - String templateName = "vanilla"; - if (params.get("template") != null) { - templateName = params.get("template"); - } + String templateName = ofNullable(params.get("template")).orElse("vanilla"); int replicationFactor = params.getInt("replicationFactor", 1); int nodeInstance = params.getInt("nodeInstance", -1); @@ -423,23 +438,25 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler return newCore(coreName, numShards, storeRef, templateName, replicationFactor, nodeInstance, numNodes, shardIds, properties, rsp); } - private boolean newDefaultCore(SolrQueryRequest req, SolrQueryResponse rsp) { - + private boolean newDefaultCore(SolrQueryRequest req, SolrQueryResponse response) + { SolrParams params = req.getParams(); String coreName = params.get("coreName") != null?params.get("coreName"):"alfresco"; - StoreRef storeRef = StoreRef.STORE_REF_WORKSPACE_SPACESSTORE; String templateName = params.get("template") != null?params.get("template"): DEFAULT_TEMPLATE; + Properties extraProperties = extractCustomProperties(params); - if (params.get("storeRef") != null) { - String store = params.get("storeRef"); - storeRef = new StoreRef(store); - } - - return newDefaultCore(coreName, storeRef, templateName, extraProperties, rsp); + return newDefaultCore( + coreName, + ofNullable(params.get("storeRef")) + .map(StoreRef::new) + .orElse(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE), + templateName, + extraProperties, + response); } - protected boolean newDefaultCore(String coreName, StoreRef storeRef, String templateName, Properties extraProperties, SolrQueryResponse rsp) + private boolean newDefaultCore(String coreName, StoreRef storeRef, String templateName, Properties extraProperties, SolrQueryResponse rsp) { return newCore(coreName, 1, storeRef, templateName, 1, 1, 1, null, extraProperties, rsp); } @@ -453,9 +470,8 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler File templates = new File(solrHome, "templates"); File template = new File(templates, templateName); - if(numShards > 1 ) + if(numShards > 1) { - String collectionName = templateName + "--" + storeRef.getProtocol() + "-" + storeRef.getIdentifier() + "--shards--"+numShards + "-x-"+replicationFactor+"--node--"+nodeInstance+"-of-"+numNodes; String coreBase = storeRef.getProtocol() + "-" + storeRef.getIdentifier() + "-"; if (coreName != null) @@ -488,18 +504,18 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler for(Integer shard : shards) { - coreName = coreBase+shard; + coreName = coreBase + shard; File newCore = new File(baseDirectory, coreName); String solrCoreName = coreName; if (coreName == null) { if(storeRef.equals(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE)) { - solrCoreName = "alfresco-"+shard; + solrCoreName = "alfresco-" + shard; } else if(storeRef.equals(StoreRef.STORE_REF_ARCHIVE_SPACESSTORE)) { - solrCoreName = "archive-"+shard; + solrCoreName = "archive-" + shard; } } createAndRegisterNewCore(rsp, extraProperties, storeRef, template, solrCoreName, newCore, numShards, shard, templateName); @@ -527,19 +543,15 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } - /** - * @param shardIds - * @return - */ private List extractShards(String shardIds, int numShards) { - ArrayList shards = new ArrayList(); + List shards = new ArrayList<>(); for(String shardId : shardIds.split(",")) { try { - Integer shard = Integer.valueOf(shardId); - if(shard.intValue() < numShards) + int shard = Integer.parseInt(shardId); + if(shard < numShards) { shards.add(shard); } @@ -552,17 +564,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler return shards; } - /** - * @param rsp - * @param storeRef - * @param template - * @param coreName - * @param newCore - * @throws IOException - * @throws FileNotFoundException - */ - private void createAndRegisterNewCore(SolrQueryResponse rsp, Properties extraProperties, StoreRef storeRef, File template, String coreName, File newCore, int shardCount, int shardInstance, String templateName) throws IOException, - FileNotFoundException + private void createAndRegisterNewCore(SolrQueryResponse rsp, Properties extraProperties, StoreRef storeRef, File template, String coreName, File newCore, int shardCount, int shardInstance, String templateName) throws IOException { if (coreContainer.getLoadedCoreNames().contains(coreName)) { @@ -613,16 +615,11 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler properties.store(fileOutputStream, null); } - SolrCore core = coreContainer.create(coreName, newCore.toPath(), new HashMap(), false); + SolrCore core = coreContainer.create(coreName, newCore.toPath(), new HashMap<>(), false); rsp.add("core", core.getName()); } - /** - * Tests to see if one of the cores is an Alfresco special core! - * @param cores - * @return - */ - public boolean hasAlfrescoCore(Collection cores) + private boolean hasAlfrescoCore(Collection cores) { if (cores == null || cores.isEmpty()) return false; for (SolrCore core:cores) @@ -632,26 +629,24 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler return false; } - private boolean updateShared(SolrQueryRequest req, SolrQueryResponse rsp) + private void updateShared(SolrQueryRequest req) { SolrParams params = req.getParams(); - try { - + try + { File config = new File(AlfrescoSolrDataModel.getResourceDirectory(), AlfrescoSolrDataModel.SHARED_PROPERTIES); updateSharedProperties(params, config, hasAlfrescoCore(coreContainer.getCores())); coreContainer.getCores().forEach(aCore -> coreContainer.reload(aCore.getName())); - - return true; - } catch (IOException e) + } + catch (IOException e) { LOGGER.error("Failed to update Shared properties ", e); } - return false; } - private boolean updateCore(SolrQueryRequest req, SolrQueryResponse rsp) + private void updateCore(SolrQueryRequest req) { String coreName = null; SolrParams params = req.getParams(); @@ -661,36 +656,28 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler coreName = params.get("coreName"); } - if ((coreName == null) || (coreName.length() == 0)) { return false; } - - SolrCore core = null; - try { - core = coreContainer.getCore(coreName); - - if(core == null) - { - return false; - } - - String configLocaltion = core.getResourceLoader().getConfigDir(); - File config = new File(configLocaltion, "solrcore.properties"); - updatePropertiesFile(params, config, null); - - coreContainer.reload(coreName); - - return true; - } - finally + if ((coreName == null) || (coreName.length() == 0)) { - //Decrement core open count - if(core != null) - { - core.close(); - } + return; } + + try (SolrCore core = coreContainer.getCore(coreName)) + { + + if (core == null) + { + return; + } + + String configLocaltion = core.getResourceLoader().getConfigDir(); + File config = new File(configLocaltion, "solrcore.properties"); + updatePropertiesFile(params, config, null); + + coreContainer.reload(coreName); + } } - private boolean removeCore(SolrQueryRequest req, SolrQueryResponse rsp) + private void removeCore(SolrQueryRequest req) { String store = ""; SolrParams params = req.getParams(); @@ -699,7 +686,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler store = params.get("storeRef"); } - if ((store == null) || (store.length() == 0)) { return false; } + if ((store == null) || (store.length() == 0)) { return; } StoreRef storeRef = new StoreRef(store); String coreName = storeRef.getProtocol() + "-" + storeRef.getIdentifier(); @@ -710,142 +697,146 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler // remove core coreContainer.unload(coreName, true, true, true); - - return true; } - - private void actionFIX(String coreName) throws AuthenticationException, IOException, JSONException, EncoderException { - // Gets Metadata health and fixes any problems - MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - IndexHealthReport indexHealthReport = metadataTracker.checkIndex(null, null, null, null); - IOpenBitSet toReindex = indexHealthReport.getTxInIndexButNotInDb(); - toReindex.or(indexHealthReport.getDuplicatedTxInIndex()); - toReindex.or(indexHealthReport.getMissingTxFromIndex()); - long current = -1; - // Goes through problems in the index - while ((current = toReindex.nextSetBit(current + 1)) != -1) + if (isMasterOrStandalone(coreName)) { - metadataTracker.addTransactionToReindex(current); - } - - // Gets the Acl health and fixes any problems - AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - indexHealthReport = aclTracker.checkIndex(null, null, null, null); - toReindex = indexHealthReport.getAclTxInIndexButNotInDb(); - toReindex.or(indexHealthReport.getDuplicatedAclTxInIndex()); - toReindex.or(indexHealthReport.getMissingAclTxFromIndex()); - current = -1; - // Goes through the problems in the index - while ((current = toReindex.nextSetBit(current + 1)) != -1) - { - aclTracker.addAclChangeSetToReindex(current); + // Gets Metadata health and fixes any problems + MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + IndexHealthReport indexHealthReport = metadataTracker.checkIndex(null, null, null, null); + IOpenBitSet toReindex = indexHealthReport.getTxInIndexButNotInDb(); + toReindex.or(indexHealthReport.getDuplicatedTxInIndex()); + toReindex.or(indexHealthReport.getMissingTxFromIndex()); + long current = -1; + // Goes through problems in the index + while ((current = toReindex.nextSetBit(current + 1)) != -1) + { + metadataTracker.addTransactionToReindex(current); + } + + // Gets the Acl health and fixes any problems + AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + indexHealthReport = aclTracker.checkIndex(null, null, null, null); + toReindex = indexHealthReport.getAclTxInIndexButNotInDb(); + toReindex.or(indexHealthReport.getDuplicatedAclTxInIndex()); + toReindex.or(indexHealthReport.getMissingAclTxFromIndex()); + current = -1; + // Goes through the problems in the index + while ((current = toReindex.nextSetBit(current + 1)) != -1) + { + aclTracker.addAclChangeSetToReindex(current); + } } } private void actionCHECK(String cname) { + trackerRegistry.getCoreNames() + .stream() + .filter(coreName -> cname == null || coreName.equals(cname)) + .map(trackerRegistry::getTrackersForCore) + .flatMap(Collection::stream) + .map(Tracker::getTrackerState) + .forEach(state -> state.setCheck(true)); + } + + private void actionACLREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws JSONException + { + NamedList report = new SimpleOrderedMap<>(); + rsp.add("report", report); + + Long aclid = + ofNullable(params.get(ARG_ACLID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_ACLID + " parameter set.")); + if (cname != null) { - for (Tracker tracker : trackerRegistry.getTrackersForCore(cname)) - { - tracker.getTrackerState().setCheck(true); - } + ofNullable(trackerRegistry.getTrackerForCore(cname, AclTracker.class)) + .ifPresent(tracker -> report.add(cname, buildAclReport(tracker, aclid))); } else { - for (String core : trackerRegistry.getCoreNames()) - { - for (Tracker tracker : trackerRegistry.getTrackersForCore(core)) - { - tracker.getTrackerState().setCheck(true); - } - } + trackerRegistry.getCoreNames() + .forEach(coreName -> + ofNullable(trackerRegistry.getTrackerForCore(coreName, AclTracker.class)) + .ifPresent(tracker -> report.add(coreName, buildAclReport(tracker, aclid)))); + } + + if (report.size() == 0) + { + report.add("WARNING", "This response comes from a slave core. Please consider to ask the same request to its corresponding master core, in order to get more information about the requested Node"); } } - private void actionACLREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws IOException, - JSONException + private void actionTXREPORT(SolrQueryResponse rsp, SolrParams params, String cname) + throws AuthenticationException, IOException, JSONException, EncoderException { - if (params.get(ARG_ACLID) == null) - { - throw new AlfrescoRuntimeException("No aclid parameter set"); - } - - if (cname != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - NamedList report = new SimpleOrderedMap(); - AclTracker tracker = trackerRegistry.getTrackerForCore(cname, AclTracker.class); - report.add(cname, buildAclReport(tracker, aclid)); - rsp.add("report", report); - } - else - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - NamedList report = new SimpleOrderedMap(); - for (String coreName : trackerRegistry.getCoreNames()) - { - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - report.add(coreName, buildAclReport(tracker, aclid)); - } - rsp.add("report", report); - } - } - - private void actionTXREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws AuthenticationException, - IOException, JSONException, EncoderException - { - if (params.get(ARG_TXID) == null) - { - throw new AlfrescoRuntimeException("No txid parameter set"); - } - if (cname == null) - { - throw new AlfrescoRuntimeException("No cname parameter set"); - } + NamedList report = new SimpleOrderedMap<>(); + rsp.add("report", report); MetadataTracker tracker = trackerRegistry.getTrackerForCore(cname, MetadataTracker.class); - Long txid = Long.valueOf(params.get(ARG_TXID)); - NamedList report = new SimpleOrderedMap(); - report.add(cname, buildTxReport(getTrackerRegistry(), informationServers.get(cname), cname, tracker, txid)); - rsp.add("report", report); + if (tracker != null) + { + Long txid = + ofNullable(params.get(ARG_TXID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_TXID + " parameter set.")); + + if (cname == null) + { + throw new AlfrescoRuntimeException("No cname parameter set"); + } + + report.add(cname, buildTxReport(getTrackerRegistry(), informationServers.get(cname), cname, tracker, txid)); + } + else + { + report.add("WARNING", "This response comes from a slave core. Please consider to ask the same request to its corresponding master core, in order to get more information about the requested Node"); + } } - private void actionACLTXREPORT(SolrQueryResponse rsp, SolrParams params, String cname) - throws AuthenticationException, IOException, JSONException, EncoderException + private void actionACLTXREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws JSONException { if (params.get(ARG_ACLTXID) == null) { throw new AlfrescoRuntimeException("No acltxid parameter set"); } - + + NamedList report = new SimpleOrderedMap<>(); + rsp.add("report", report); + + Long acltxid = + ofNullable(params.get(ARG_ACLTXID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_ACLTXID + " parameter set.")); + if (cname != null) { - AclTracker tracker = trackerRegistry.getTrackerForCore(cname, AclTracker.class); - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - NamedList report = new SimpleOrderedMap(); - report.add(cname, buildAclTxReport(getTrackerRegistry(), informationServers.get(cname), cname, tracker, acltxid)); - rsp.add("report", report); + ofNullable(trackerRegistry.getTrackerForCore(cname, AclTracker.class)) + .ifPresent(tracker -> report.add(cname, buildAclTxReport(trackerRegistry, informationServers.get(cname), cname, tracker, acltxid))); } else { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - NamedList report = new SimpleOrderedMap(); - for (String coreName : trackerRegistry.getCoreNames()) - { - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - report.add(coreName, buildAclTxReport(getTrackerRegistry(), informationServers.get(coreName), coreName, tracker, acltxid)); - } - rsp.add("report", report); + trackerRegistry.getCoreNames() + .forEach(coreName -> + ofNullable(trackerRegistry.getTrackerForCore(coreName, AclTracker.class)) + .ifPresent(tracker -> report.add(cname, buildAclTxReport(trackerRegistry, informationServers.get(cname), cname, tracker, acltxid)))); + } + + if (report.size() == 0) + { + report.add("WARNING", "This response comes from a slave core. Please consider to ask the same request to its corresponding master core, in order to get more information about the requested Node"); } } - private void actionREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws IOException, - JSONException, AuthenticationException, EncoderException + private void actionREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws JSONException { + NamedList report = new SimpleOrderedMap<>(); + rsp.add("report", report); + Long fromTime = getSafeLong(params, "fromTime"); Long toTime = getSafeLong(params, "toTime"); Long fromTx = getSafeLong(params, "fromTx"); @@ -855,32 +846,17 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler if (cname != null) { - NamedList report = new SimpleOrderedMap(); - if (trackerRegistry.hasTrackersForCore(cname)) + if (trackerRegistry.hasTrackersForCore(cname) && isMasterOrStandalone(cname)) { - report.add(cname, buildTrackerReport(getTrackerRegistry(), informationServers.get(cname),cname, fromTx, toTx, fromAclTx, toAclTx, fromTime, toTime)); - rsp.add("report", report); - } - else - { - report.add(cname, "Core unknown"); + report.add(cname, buildTrackerReport(trackerRegistry, informationServers.get(cname),cname, fromTx, toTx, fromAclTx, toAclTx, fromTime, toTime)); } } else { - NamedList report = new SimpleOrderedMap(); - for (String coreName : trackerRegistry.getCoreNames()) - { - if (trackerRegistry.hasTrackersForCore(coreName)) - { - report.add(coreName, buildTrackerReport(getTrackerRegistry(), informationServers.get(coreName), coreName, fromTx, toTx, fromAclTx, toAclTx, fromTime, toTime)); - } - else - { - report.add(coreName, "Core unknown"); - } - } - rsp.add("report", report); + trackerRegistry.getCoreNames().stream() + .filter(trackerRegistry::hasTrackersForCore) + .filter(this::isMasterOrStandalone) + .forEach(coreName -> report.add(coreName, buildTrackerReport(trackerRegistry, informationServers.get(coreName), coreName, fromTx, toTx, fromAclTx, toAclTx, fromTime, toTime))); } } @@ -896,8 +872,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } - DocRouter docRouter = metadataTracker.getDocRouter(); - return docRouter; + return metadataTracker.getDocRouter(); } @@ -907,8 +882,8 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler DocRouter docRouter = getDocRouter(cname); - if(docRouter instanceof DBIDRangeRouter) { - + if(docRouter instanceof DBIDRangeRouter) + { DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter) docRouter; if(!dbidRangeRouter.getInitialized()) @@ -938,7 +913,8 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler double density = 0; - if(offset > 0) { + if(offset > 0) + { density = ((double)nodeCount) / ((double)offset); // This is how dense we are so far. } @@ -975,15 +951,15 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler rsp.add("density", Math.abs(density)); rsp.add("expand", bestGuess); rsp.add("expanded", dbidRangeRouter.getExpanded()); - } else { + } + else + { rsp.add("expand", -1); rsp.add("exception", "ERROR: Wrong document router type:"+docRouter.getClass().getSimpleName()); - return; } } - private synchronized void expand(SolrQueryResponse rsp, SolrParams params, String cname) - throws IOException + private synchronized void expand(SolrQueryResponse rsp, SolrParams params, String cname) throws IOException { InformationServer informationServer = informationServers.get(cname); DocRouter docRouter = getDocRouter(cname); @@ -1011,7 +987,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler long startRange = dbidRangeRouter.getStartRange(); long maxNodeId = informationServer.maxNodeId(); - long range = currentEndRange-startRange; + long range = currentEndRange - startRange; long safe = startRange + ((long) (range * .75)); if(maxNodeId > safe) @@ -1030,65 +1006,38 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler dbidRangeRouter.setExpanded(true); assert newEndRange == dbidRangeRouter.getEndRange(); rsp.add("expand", dbidRangeRouter.getEndRange()); - return; } catch(Throwable t) { rsp.add("expand", -1); rsp.add("exception", t.getMessage()); LOGGER.error("exception expanding", t); - return; } } else { rsp.add("expand", -1); - rsp.add("exception", "Wrong document router type:"+docRouter.getClass().getSimpleName()); - return; + rsp.add("exception", "Wrong document router type:" + docRouter.getClass().getSimpleName()); } } - private void actionNODEREPORTS(SolrQueryResponse rsp, SolrParams params, String cname) throws IOException, - JSONException + private void actionNODEREPORTS(SolrQueryResponse rsp, SolrParams params, String cname) throws JSONException { + Long dbid = + ofNullable(params.get(ARG_NODEID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No dbid parameter set.")); + + NamedList report = new SimpleOrderedMap<>(); + rsp.add("report", report); + if (cname != null) { - MetadataTracker tracker = trackerRegistry.getTrackerForCore(cname, - MetadataTracker.class); - Long dbid = null; - if (params.get(ARG_NODEID) != null) - { - dbid = Long.valueOf(params.get(ARG_NODEID)); - NamedList report = new SimpleOrderedMap(); - report.add(cname, buildNodeReport(tracker, dbid)); - rsp.add("report", report); - - } - else - { - throw new AlfrescoRuntimeException("No dbid parameter set"); - } + report.add(cname, buildNodeReport(nodeStatePublisher(cname), dbid)); } else { - Long dbid = null; - if (params.get(ARG_NODEID) != null) - { - dbid = Long.valueOf(params.get(ARG_NODEID)); - NamedList report = new SimpleOrderedMap(); - for (String coreName : trackerRegistry.getCoreNames()) - { - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, - MetadataTracker.class); - report.add(coreName, buildNodeReport(tracker, dbid)); - } - rsp.add("report", report); - } - else - { - throw new AlfrescoRuntimeException("No dbid parameter set"); - } - + trackerRegistry.getCoreNames().forEach(coreName -> report.add(coreName, buildNodeReport(nodeStatePublisher(coreName), dbid))); } } @@ -1102,11 +1051,18 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler InformationServer srv = informationServers.get(coreName); if (srv != null) { - addCoreSummary(trackerRegistry, coreName, detail, hist, values, srv, report); - - if (reset) + if (isMasterOrStandalone(coreName)) { - srv.getTrackerStats().reset(); + addMasterOrStandaloneCoreSummary(trackerRegistry, coreName, detail, hist, values, srv, report); + + if (reset) + { + srv.getTrackerStats().reset(); + } + } + else + { + addSlaveCoreSummary(trackerRegistry, coreName, detail, hist, values, srv, report); } } else @@ -1115,106 +1071,128 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } - private void actionINDEX(SolrParams params, String coreName) { - if (params.get(ARG_TXID) != null) + if (isMasterOrStandalone(coreName)) { - Long txid = Long.valueOf(params.get(ARG_TXID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addTransactionToIndex(txid); - } - if (params.get(ARG_ACLTXID) != null) - { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclChangeSetToIndex(acltxid); - } - if (params.get(ARG_NODEID) != null) - { - Long nodeid = Long.valueOf(params.get(ARG_NODEID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addNodeToIndex(nodeid); - } - if (params.get(ARG_ACLID) != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclToIndex(aclid); + if (params.get(ARG_TXID) != null) + { + Long txid = Long.valueOf(params.get(ARG_TXID)); + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + tracker.addTransactionToIndex(txid); + } + + if (params.get(ARG_ACLTXID) != null) + { + Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); + AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + tracker.addAclChangeSetToIndex(acltxid); + } + + if (params.get(ARG_NODEID) != null) + { + Long nodeid = Long.valueOf(params.get(ARG_NODEID)); + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + tracker.addNodeToIndex(nodeid); + } + + if (params.get(ARG_ACLID) != null) + { + Long aclid = Long.valueOf(params.get(ARG_ACLID)); + AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + tracker.addAclToIndex(aclid); + } } } private void actionRETRY(SolrQueryResponse rsp, String coreName) throws IOException { - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - InformationServer srv = informationServers.get(coreName); - Set errorDocIds = srv.getErrorDocIds(); - for (Long nodeid : errorDocIds) + if (isMasterOrStandalone(coreName)) { - tracker.addNodeToReindex(nodeid); + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + InformationServer srv = informationServers.get(coreName); + + Set errorDocIds = srv.getErrorDocIds(); + for (Long nodeid : errorDocIds) + { + tracker.addNodeToReindex(nodeid); + } + rsp.add(coreName, errorDocIds); } - rsp.add(coreName, errorDocIds); } private void actionREINDEX(SolrParams params, String coreName) { - if (params.get(ARG_TXID) != null) + if (isMasterOrStandalone(coreName)) { - Long txid = Long.valueOf(params.get(ARG_TXID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addTransactionToReindex(txid); - } - if (params.get(ARG_ACLTXID) != null) - { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclChangeSetToReindex(acltxid); - } - if (params.get(ARG_NODEID) != null) - { - Long nodeid = Long.valueOf(params.get(ARG_NODEID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addNodeToReindex(nodeid); - } - if (params.get(ARG_ACLID) != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclToReindex(aclid); - } - if (params.get(ARG_QUERY) != null) - { - String query = params.get(ARG_QUERY); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addQueryToReindex(query); + if (params.get(ARG_TXID) != null) + { + Long txid = Long.valueOf(params.get(ARG_TXID)); + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + tracker.addTransactionToReindex(txid); + } + + if (params.get(ARG_ACLTXID) != null) + { + Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); + AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + tracker.addAclChangeSetToReindex(acltxid); + } + + if (params.get(ARG_NODEID) != null) + { + Long nodeid = Long.valueOf(params.get(ARG_NODEID)); + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + tracker.addNodeToReindex(nodeid); + } + + if (params.get(ARG_ACLID) != null) + { + Long aclid = Long.valueOf(params.get(ARG_ACLID)); + AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + tracker.addAclToReindex(aclid); + } + + if (params.get(ARG_QUERY) != null) + { + String query = params.get(ARG_QUERY); + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + tracker.addQueryToReindex(query); + } } } private void actionPURGE(SolrParams params, String coreName) { - if (params.get(ARG_TXID) != null) + if (isMasterOrStandalone(coreName)) { - Long txid = Long.valueOf(params.get(ARG_TXID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addTransactionToPurge(txid); - } - if (params.get(ARG_ACLTXID) != null) - { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclChangeSetToPurge(acltxid); - } - if (params.get(ARG_NODEID) != null) - { - Long nodeid = Long.valueOf(params.get(ARG_NODEID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addNodeToPurge(nodeid); - } - if (params.get(ARG_ACLID) != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclToPurge(aclid); + if (params.get(ARG_TXID) != null) + { + Long txid = Long.valueOf(params.get(ARG_TXID)); + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + tracker.addTransactionToPurge(txid); + } + + if (params.get(ARG_ACLTXID) != null) + { + Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); + AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + tracker.addAclChangeSetToPurge(acltxid); + } + + if (params.get(ARG_NODEID) != null) + { + Long nodeid = Long.valueOf(params.get(ARG_NODEID)); + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + tracker.addNodeToPurge(nodeid); + } + + if (params.get(ARG_ACLID) != null) + { + Long aclid = Long.valueOf(params.get(ARG_ACLID)); + AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + tracker.addAclToPurge(aclid); + } } } @@ -1228,7 +1206,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler return trackerRegistry; } - protected void setTrackerRegistry(TrackerRegistry trackerRegistry) + void setTrackerRegistry(TrackerRegistry trackerRegistry) { this.trackerRegistry = trackerRegistry; } @@ -1237,4 +1215,34 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler { return scheduler; } + + private void waitForTenSeconds() + { + try + { + TimeUnit.SECONDS.sleep(10); + } + catch (InterruptedException e) + { + //Don't care + } + } + + /** + * Returns, for the given core, the component which is in charge to publish the core state. + * + * @param coreName the owning core name. + * @return the component which is in charge to publish the core state. + */ + private CoreStatePublisher nodeStatePublisher(String coreName) + { + return ofNullable(trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class)) + .map(CoreStatePublisher.class::cast) + .orElse(trackerRegistry.getTrackerForCore(coreName, SlaveCoreStatePublisher.class)); + } + + private boolean isMasterOrStandalone(String coreName) + { + return trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class) != null; + } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java index 5ccd37ed8..8aca5f99c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java @@ -21,6 +21,7 @@ */ package org.alfresco.solr; +import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.AuthenticationException; import org.alfresco.service.cmr.repository.datatype.Duration; import org.alfresco.solr.client.Node; @@ -37,24 +38,18 @@ import java.util.List; import java.util.Map; import java.util.Set; +import static java.util.Optional.ofNullable; + /** * Methods taken from AlfrescoCoreAdminHandler that deal with building reports */ -public class HandlerReportBuilder { - - /** - * Builds AclReport - * @param tracker - * @param aclid - * @return - * @throws IOException - * @throws JSONException - */ - public static NamedList buildAclReport(AclTracker tracker, Long aclid) throws IOException, JSONException +class HandlerReportBuilder +{ + static NamedList buildAclReport(AclTracker tracker, Long aclid) throws JSONException { AclReport aclReport = tracker.checkAcl(aclid); - NamedList nr = new SimpleOrderedMap(); + NamedList nr = new SimpleOrderedMap<>(); nr.add("Acl Id", aclReport.getAclId()); nr.add("Acl doc in index", aclReport.getIndexAclDoc()); if (aclReport.getIndexAclDoc() != null) @@ -65,26 +60,14 @@ public class HandlerReportBuilder { return nr; } - /** - * Builds TxReport - * @param trackerRegistry - * @param srv - * @param coreName - * @param tracker - * @param txid - * @return - * @throws AuthenticationException - * @throws IOException - * @throws JSONException - * @throws EncoderException - */ - public static NamedList buildTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, MetadataTracker tracker, Long txid) + static NamedList buildTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, MetadataTracker tracker, Long txid) throws AuthenticationException, IOException, JSONException, EncoderException { - NamedList nr = new SimpleOrderedMap(); + NamedList nr = new SimpleOrderedMap<>(); nr.add("TXID", txid); - nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, txid, txid, 0l, 0l, null, null)); - NamedList nodes = new SimpleOrderedMap(); + nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, txid, txid, 0L, 0L, null, null)); + NamedList nodes = new SimpleOrderedMap<>(); + // add node reports .... List dbNodes = tracker.getFullNodesForDbTransaction(txid); for (Node node : dbNodes) @@ -97,50 +80,34 @@ public class HandlerReportBuilder { return nr; } - /** - * Builds AclTxReport - * @param trackerRegistry - * @param srv - * @param coreName - * @param tracker - * @param acltxid - * @return - * @throws AuthenticationException - * @throws IOException - * @throws JSONException - * @throws EncoderException - */ - public static NamedList buildAclTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, AclTracker tracker, Long acltxid) - throws AuthenticationException, IOException, JSONException, EncoderException + static NamedList buildAclTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, AclTracker tracker, Long acltxid) throws JSONException { - NamedList nr = new SimpleOrderedMap(); - nr.add("TXID", acltxid); - nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, 0l, 0l, acltxid, acltxid, null, null)); - NamedList nodes = new SimpleOrderedMap(); - // add node reports .... - List dbAclIds = tracker.getAclsForDbAclTransaction(acltxid); - for (Long aclid : dbAclIds) + try { + NamedList nr = new SimpleOrderedMap<>(); + nr.add("TXID", acltxid); + nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, 0L, 0L, acltxid, acltxid, null, null)); + NamedList nodes = new SimpleOrderedMap<>(); + + // add node reports .... + List dbAclIds = tracker.getAclsForDbAclTransaction(acltxid); + for (Long aclid : dbAclIds) { + nodes.add("ACLID " + aclid, buildAclReport(tracker, aclid)); + } + nr.add("aclTxDbAclCount", dbAclIds.size()); + nr.add("nodes", nodes); + return nr; + } + catch (Exception exception) { - nodes.add("ACLID " + aclid, buildAclReport(tracker, aclid)); + throw new AlfrescoRuntimeException("", exception); } - nr.add("aclTxDbAclCount", dbAclIds.size()); - nr.add("nodes", nodes); - return nr; } - /** - * Builds Node report - * @param tracker - * @param node - * @return - * @throws IOException - * @throws JSONException - */ - public static NamedList buildNodeReport(MetadataTracker tracker, Node node) throws IOException, JSONException + static NamedList buildNodeReport(MetadataTracker tracker, Node node) throws JSONException { NodeReport nodeReport = tracker.checkNode(node); - NamedList nr = new SimpleOrderedMap(); + NamedList nr = new SimpleOrderedMap<>(); nr.add("Node DBID", nodeReport.getDbid()); nr.add("DB TX", nodeReport.getDbTx()); nr.add("DB TX status", nodeReport.getDbNodeStatus().toString()); @@ -156,161 +123,258 @@ public class HandlerReportBuilder { return nr; } - /** - * Builds Node Report - * @param tracker - * @param dbid - * @return - * @throws IOException - * @throws JSONException - */ - public static NamedList buildNodeReport(MetadataTracker tracker, Long dbid) throws IOException, JSONException + static NamedList buildNodeReport(CoreStatePublisher publisher, Long dbid) throws JSONException { - NodeReport nodeReport = tracker.checkNode(dbid); + NodeReport nodeReport = publisher.checkNode(dbid); - NamedList nr = new SimpleOrderedMap(); - nr.add("Node DBID", nodeReport.getDbid()); - nr.add("DB TX", nodeReport.getDbTx()); - nr.add("DB TX status", nodeReport.getDbNodeStatus().toString()); - if (nodeReport.getIndexLeafDoc() != null) + NamedList payload = new SimpleOrderedMap<>(); + payload.add("Node DBID", nodeReport.getDbid()); + + if (publisher.isOnMasterOrStandalone()) { - nr.add("Leaf tx in Index", nodeReport.getIndexLeafTx()); + ofNullable(nodeReport.getDbTx()).ifPresent(value -> payload.add("DB TX", value)); + ofNullable(nodeReport.getDbNodeStatus()).map(Object::toString).ifPresent(value -> payload.add("DB TX Status", value)); + ofNullable(nodeReport.getIndexLeafTx()).ifPresent(value -> payload.add("Leaf tx in Index", value)); + ofNullable(nodeReport.getIndexAuxDoc()).ifPresent(value -> payload.add("Aux tx in Index", value)); } - if (nodeReport.getIndexAuxDoc() != null) + else { - nr.add("Aux tx in Index", nodeReport.getIndexAuxTx()); + payload.add("WARNING", "This response comes from a slave core. Please consider to ask the same to its corresponding master core, in order to get more information about the requested Node"); } - nr.add("Indexed Node Doc Count", nodeReport.getIndexedNodeDocCount()); - return nr; + + ofNullable(nodeReport.getIndexedNodeDocCount()).ifPresent(value -> payload.add("Indexed Node Doc Count", value)); + + return payload; } /** * Builds Tracker report - * @param trackerRegistry - * @param srv - * @param coreName - * @param fromTx - * @param toTx - * @param fromAclTx - * @param toAclTx - * @param fromTime - * @param toTime - * @return - * @throws IOException - * @throws JSONException - * @throws AuthenticationException - * @throws EncoderException */ - public static NamedList buildTrackerReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, Long fromTx, Long toTx, Long fromAclTx, Long toAclTx, - Long fromTime, Long toTime) throws IOException, JSONException, AuthenticationException, EncoderException + static NamedList buildTrackerReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, Long fromTx, Long toTx, Long fromAclTx, Long toAclTx, + Long fromTime, Long toTime) throws JSONException { - // ACL - AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - IndexHealthReport aclReport = aclTracker.checkIndex(toTx, toAclTx, fromTime, toTime); - NamedList ihr = new SimpleOrderedMap(); - ihr.add("Alfresco version", aclTracker.getAlfrescoVersion()); - ihr.add("DB acl transaction count", aclReport.getDbAclTransactionCount()); - ihr.add("Count of duplicated acl transactions in the index", aclReport.getDuplicatedAclTxInIndex() - .cardinality()); - if (aclReport.getDuplicatedAclTxInIndex().cardinality() > 0) + try { - ihr.add("First duplicate acl tx", aclReport.getDuplicatedAclTxInIndex().nextSetBit(0L)); - } - ihr.add("Count of acl transactions in the index but not the DB", aclReport.getAclTxInIndexButNotInDb() - .cardinality()); - if (aclReport.getAclTxInIndexButNotInDb().cardinality() > 0) - { - ihr.add("First acl transaction in the index but not the DB", aclReport.getAclTxInIndexButNotInDb() - .nextSetBit(0L)); - } - ihr.add("Count of missing acl transactions from the Index", aclReport.getMissingAclTxFromIndex() - .cardinality()); - if (aclReport.getMissingAclTxFromIndex().cardinality() > 0) - { - ihr.add("First acl transaction missing from the Index", aclReport.getMissingAclTxFromIndex() - .nextSetBit(0L)); - } - ihr.add("Index acl transaction count", aclReport.getAclTransactionDocsInIndex()); - ihr.add("Index unique acl transaction count", aclReport.getAclTransactionDocsInIndex()); - TrackerState aclState = aclTracker.getTrackerState(); - ihr.add("Last indexed change set commit time", aclState.getLastIndexedChangeSetCommitTime()); - Date lastChangeSetDate = new Date(aclState.getLastIndexedChangeSetCommitTime()); - ihr.add("Last indexed change set commit date", CachingDateFormat.getDateFormat().format(lastChangeSetDate)); - ihr.add("Last changeset id before holes", aclState.getLastIndexedChangeSetIdBeforeHoles()); + // ACL + AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + IndexHealthReport aclReport = aclTracker.checkIndex(toTx, toAclTx, fromTime, toTime); + NamedList ihr = new SimpleOrderedMap<>(); + ihr.add("Alfresco version", aclTracker.getAlfrescoVersion()); + ihr.add("DB acl transaction count", aclReport.getDbAclTransactionCount()); + ihr.add("Count of duplicated acl transactions in the index", aclReport.getDuplicatedAclTxInIndex() + .cardinality()); + if (aclReport.getDuplicatedAclTxInIndex().cardinality() > 0) { + ihr.add("First duplicate acl tx", aclReport.getDuplicatedAclTxInIndex().nextSetBit(0L)); + } + ihr.add("Count of acl transactions in the index but not the DB", aclReport.getAclTxInIndexButNotInDb() + .cardinality()); + if (aclReport.getAclTxInIndexButNotInDb().cardinality() > 0) { + ihr.add("First acl transaction in the index but not the DB", aclReport.getAclTxInIndexButNotInDb() + .nextSetBit(0L)); + } + ihr.add("Count of missing acl transactions from the Index", aclReport.getMissingAclTxFromIndex() + .cardinality()); + if (aclReport.getMissingAclTxFromIndex().cardinality() > 0) { + ihr.add("First acl transaction missing from the Index", aclReport.getMissingAclTxFromIndex() + .nextSetBit(0L)); + } + ihr.add("Index acl transaction count", aclReport.getAclTransactionDocsInIndex()); + ihr.add("Index unique acl transaction count", aclReport.getAclTransactionDocsInIndex()); + TrackerState aclState = aclTracker.getTrackerState(); + ihr.add("Last indexed change set commit time", aclState.getLastIndexedChangeSetCommitTime()); + Date lastChangeSetDate = new Date(aclState.getLastIndexedChangeSetCommitTime()); + ihr.add("Last indexed change set commit date", CachingDateFormat.getDateFormat().format(lastChangeSetDate)); + ihr.add("Last changeset id before holes", aclState.getLastIndexedChangeSetIdBeforeHoles()); - // Metadata - MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, toAclTx, fromTime, toTime); - ihr.add("DB transaction count", metaReport.getDbTransactionCount()); - ihr.add("Count of duplicated transactions in the index", metaReport.getDuplicatedTxInIndex() - .cardinality()); - if (metaReport.getDuplicatedTxInIndex().cardinality() > 0) - { - ihr.add("First duplicate", metaReport.getDuplicatedTxInIndex().nextSetBit(0L)); - } - ihr.add("Count of transactions in the index but not the DB", metaReport.getTxInIndexButNotInDb() - .cardinality()); - if (metaReport.getTxInIndexButNotInDb().cardinality() > 0) - { - ihr.add("First transaction in the index but not the DB", metaReport.getTxInIndexButNotInDb() - .nextSetBit(0L)); - } - ihr.add("Count of missing transactions from the Index", metaReport.getMissingTxFromIndex().cardinality()); - if (metaReport.getMissingTxFromIndex().cardinality() > 0) - { - ihr.add("First transaction missing from the Index", metaReport.getMissingTxFromIndex() - .nextSetBit(0L)); - } - ihr.add("Index transaction count", metaReport.getTransactionDocsInIndex()); - ihr.add("Index unique transaction count", metaReport.getTransactionDocsInIndex()); - ihr.add("Index node count", metaReport.getLeafDocCountInIndex()); - ihr.add("Count of duplicate nodes in the index", metaReport.getDuplicatedLeafInIndex().cardinality()); - if (metaReport.getDuplicatedLeafInIndex().cardinality() > 0) - { - ihr.add("First duplicate node id in the index", metaReport.getDuplicatedLeafInIndex().nextSetBit(0L)); - } - ihr.add("Index error count", metaReport.getErrorDocCountInIndex()); - ihr.add("Count of duplicate error docs in the index", metaReport.getDuplicatedErrorInIndex() - .cardinality()); - if (metaReport.getDuplicatedErrorInIndex().cardinality() > 0) - { - ihr.add("First duplicate error in the index", SolrInformationServer.PREFIX_ERROR - + metaReport.getDuplicatedErrorInIndex().nextSetBit(0L)); - } - ihr.add("Index unindexed count", metaReport.getUnindexedDocCountInIndex()); - ihr.add("Count of duplicate unindexed docs in the index", metaReport.getDuplicatedUnindexedInIndex() - .cardinality()); - if (metaReport.getDuplicatedUnindexedInIndex().cardinality() > 0) - { - ihr.add("First duplicate unindexed in the index", - metaReport.getDuplicatedUnindexedInIndex().nextSetBit(0L)); - } - TrackerState metaState = metadataTracker.getTrackerState(); - ihr.add("Last indexed transaction commit time", metaState.getLastIndexedTxCommitTime()); - Date lastTxDate = new Date(metaState.getLastIndexedTxCommitTime()); - ihr.add("Last indexed transaction commit date", CachingDateFormat.getDateFormat().format(lastTxDate)); - ihr.add("Last TX id before holes", metaState.getLastIndexedTxIdBeforeHoles()); + // Metadata + MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, toAclTx, fromTime, toTime); + ihr.add("DB transaction count", metaReport.getDbTransactionCount()); + ihr.add("Count of duplicated transactions in the index", metaReport.getDuplicatedTxInIndex() + .cardinality()); + if (metaReport.getDuplicatedTxInIndex().cardinality() > 0) { + ihr.add("First duplicate", metaReport.getDuplicatedTxInIndex().nextSetBit(0L)); + } + ihr.add("Count of transactions in the index but not the DB", metaReport.getTxInIndexButNotInDb() + .cardinality()); + if (metaReport.getTxInIndexButNotInDb().cardinality() > 0) { + ihr.add("First transaction in the index but not the DB", metaReport.getTxInIndexButNotInDb() + .nextSetBit(0L)); + } + ihr.add("Count of missing transactions from the Index", metaReport.getMissingTxFromIndex().cardinality()); + if (metaReport.getMissingTxFromIndex().cardinality() > 0) { + ihr.add("First transaction missing from the Index", metaReport.getMissingTxFromIndex() + .nextSetBit(0L)); + } + ihr.add("Index transaction count", metaReport.getTransactionDocsInIndex()); + ihr.add("Index unique transaction count", metaReport.getTransactionDocsInIndex()); + ihr.add("Index node count", metaReport.getLeafDocCountInIndex()); + ihr.add("Count of duplicate nodes in the index", metaReport.getDuplicatedLeafInIndex().cardinality()); + if (metaReport.getDuplicatedLeafInIndex().cardinality() > 0) { + ihr.add("First duplicate node id in the index", metaReport.getDuplicatedLeafInIndex().nextSetBit(0L)); + } + ihr.add("Index error count", metaReport.getErrorDocCountInIndex()); + ihr.add("Count of duplicate error docs in the index", metaReport.getDuplicatedErrorInIndex() + .cardinality()); + if (metaReport.getDuplicatedErrorInIndex().cardinality() > 0) { + ihr.add("First duplicate error in the index", SolrInformationServer.PREFIX_ERROR + + metaReport.getDuplicatedErrorInIndex().nextSetBit(0L)); + } + ihr.add("Index unindexed count", metaReport.getUnindexedDocCountInIndex()); + ihr.add("Count of duplicate unindexed docs in the index", metaReport.getDuplicatedUnindexedInIndex() + .cardinality()); + if (metaReport.getDuplicatedUnindexedInIndex().cardinality() > 0) { + ihr.add("First duplicate unindexed in the index", + metaReport.getDuplicatedUnindexedInIndex().nextSetBit(0L)); + } + TrackerState metaState = metadataTracker.getTrackerState(); + ihr.add("Last indexed transaction commit time", metaState.getLastIndexedTxCommitTime()); + Date lastTxDate = new Date(metaState.getLastIndexedTxCommitTime()); + ihr.add("Last indexed transaction commit date", CachingDateFormat.getDateFormat().format(lastTxDate)); + ihr.add("Last TX id before holes", metaState.getLastIndexedTxIdBeforeHoles()); - srv.addFTSStatusCounts(ihr); + srv.addFTSStatusCounts(ihr); - return ihr; + return ihr; + } + catch (Exception exception) + { + throw new AlfrescoRuntimeException("", exception); + } } - - /** - * Adds a core summary - * @param cname - * @param detail - * @param hist - * @param values - * @param srv - * @param report - * @throws IOException - */ - public static void addCoreSummary(TrackerRegistry trackerRegistry, String cname, boolean detail, boolean hist, boolean values, - InformationServer srv, NamedList report) throws IOException + static void addSlaveCoreSummary(TrackerRegistry trackerRegistry, String cname, boolean detail, boolean hist, boolean values, + InformationServer srv, NamedList report) throws IOException { - NamedList coreSummary = new SimpleOrderedMap(); + NamedList coreSummary = new SimpleOrderedMap<>(); + coreSummary.addAll((SimpleOrderedMap) srv.getCoreStats()); + + SlaveCoreStatePublisher statePublisher = trackerRegistry.getTrackerForCore(cname, SlaveCoreStatePublisher.class); + TrackerState trackerState = statePublisher.getTrackerState(); + long lastIndexTxCommitTime = trackerState.getLastIndexedTxCommitTime(); + + long lastIndexedTxId = trackerState.getLastIndexedTxId(); + long lastTxCommitTimeOnServer = trackerState.getLastTxCommitTimeOnServer(); + long lastTxIdOnServer = trackerState.getLastTxIdOnServer(); + + Date lastIndexTxCommitDate = new Date(lastIndexTxCommitTime); + Date lastTxOnServerDate = new Date(lastTxCommitTimeOnServer); + long transactionsToDo = lastTxIdOnServer - lastIndexedTxId; + if (transactionsToDo < 0) + { + transactionsToDo = 0; + } + + long nodesToDo = 0; + long remainingTxTimeMillis = 0; + if (transactionsToDo > 0) + { + // We now use the elapsed time as seen by the single thread farming out metadata indexing + double meanDocsPerTx = srv.getTrackerStats().getMeanDocsPerTx(); + double meanNodeElaspedIndexTime = srv.getTrackerStats().getMeanNodeElapsedIndexTime(); + nodesToDo = (long)(transactionsToDo * meanDocsPerTx); + remainingTxTimeMillis = (long) (nodesToDo * meanNodeElaspedIndexTime); + } + Date now = new Date(); + Date end = new Date(now.getTime() + remainingTxTimeMillis); + Duration remainingTx = new Duration(now, end); + + long remainingChangeSetTimeMillis = 0; + + now = new Date(); + end = new Date(now.getTime() + remainingChangeSetTimeMillis); + Duration remainingChangeSet = new Duration(now, end); + + NamedList ftsSummary = new SimpleOrderedMap<>(); + long remainingContentTimeMillis = 0; + srv.addFTSStatusCounts(ftsSummary); + long cleanCount = + ofNullable(ftsSummary.get("Node count with FTSStatus Clean")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + long dirtyCount = + ofNullable(ftsSummary.get("Node count with FTSStatus Dirty")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + long newCount = + ofNullable(ftsSummary.get("Node count with FTSStatus New")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + + long nodesInIndex = + ofNullable(coreSummary.get("Alfresco Nodes in Index")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + + long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount + newCount)/nodesInIndex : 0; + if (dirtyCount + newCount + contentYetToSee > 0) + { + // We now use the elapsed time as seen by the single thread farming out alc indexing + double meanContentElapsedIndexTime = srv.getTrackerStats().getMeanContentElapsedIndexTime(); + remainingContentTimeMillis = (long) ((dirtyCount + newCount + contentYetToSee) * meanContentElapsedIndexTime); + } + now = new Date(); + end = new Date(now.getTime() + remainingContentTimeMillis); + Duration remainingContent = new Duration(now, end); + coreSummary.add("FTS",ftsSummary); + + Duration txLag = new Duration(lastIndexTxCommitDate, lastTxOnServerDate); + if (lastIndexTxCommitDate.compareTo(lastTxOnServerDate) > 0) + { + txLag = new Duration(); + } + long txLagSeconds = (lastTxCommitTimeOnServer - lastIndexTxCommitTime) / 1000; + if (txLagSeconds < 0) + { + txLagSeconds = 0; + } + + ModelTracker modelTrkr = trackerRegistry.getModelTracker(); + TrackerState modelTrkrState = modelTrkr.getTrackerState(); + coreSummary.add("ModelTracker Active", modelTrkrState.isRunning()); + coreSummary.add("NodeState Publisher Active", trackerState.isRunning()); + + // TX + + coreSummary.add("Last Index TX Commit Time", lastIndexTxCommitTime); + coreSummary.add("Last Index TX Commit Date", lastIndexTxCommitDate); + coreSummary.add("TX Lag", txLagSeconds + " s"); + coreSummary.add("TX Duration", txLag.toString()); + coreSummary.add("Timestamp for last TX on server", lastTxCommitTimeOnServer); + coreSummary.add("Date for last TX on server", lastTxOnServerDate); + coreSummary.add("Id for last TX on server", lastTxIdOnServer); + coreSummary.add("Id for last TX in index", lastIndexedTxId); + coreSummary.add("Approx transactions remaining", transactionsToDo); + coreSummary.add("Approx transaction indexing time remaining", remainingTx.largestComponentformattedString()); + // Stats + + coreSummary.add("Model sync times (ms)", srv.getTrackerStats().getModelTimes().getNamedList(detail, hist, values)); + coreSummary.add("Docs/Tx", srv.getTrackerStats().getTxDocs().getNamedList(detail, hist, values)); + + // Model + + Map> modelErrors = srv.getModelErrors(); + if (modelErrors.size() > 0) + { + NamedList errorList = new SimpleOrderedMap<>(); + for (Map.Entry> modelNameToErrors : modelErrors.entrySet()) + { + errorList.add(modelNameToErrors.getKey(), modelNameToErrors.getValue()); + } + coreSummary.add("Model changes are not compatible with the existing data model and have not been applied", errorList); + } + + report.add(cname, coreSummary); + } + + static void addMasterOrStandaloneCoreSummary(TrackerRegistry trackerRegistry, String cname, boolean detail, boolean hist, boolean values, + InformationServer srv, NamedList report) throws IOException + { + NamedList coreSummary = new SimpleOrderedMap<>(); coreSummary.addAll((SimpleOrderedMap) srv.getCoreStats()); MetadataTracker metaTrkr = trackerRegistry.getTrackerForCore(cname, MetadataTracker.class); @@ -368,14 +432,32 @@ public class HandlerReportBuilder { end = new Date(now.getTime() + remainingChangeSetTimeMillis); Duration remainingChangeSet = new Duration(now, end); - NamedList ftsSummary = new SimpleOrderedMap(); + NamedList ftsSummary = new SimpleOrderedMap<>(); long remainingContentTimeMillis = 0; srv.addFTSStatusCounts(ftsSummary); - long cleanCount = ((Long)ftsSummary.get("Node count with FTSStatus Clean")).longValue(); - long dirtyCount = ((Long)ftsSummary.get("Node count with FTSStatus Dirty")).longValue(); - long newCount = ((Long)ftsSummary.get("Node count with FTSStatus New")).longValue(); - long nodesInIndex = ((Long)coreSummary.get("Alfresco Nodes in Index")); - long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount + newCount)/nodesInIndex : 0;; + long cleanCount = + ofNullable(ftsSummary.get("Node count with FTSStatus Clean")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + long dirtyCount = + ofNullable(ftsSummary.get("Node count with FTSStatus Dirty")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + long newCount = + ofNullable(ftsSummary.get("Node count with FTSStatus New")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + + long nodesInIndex = + ofNullable(coreSummary.get("Alfresco Nodes in Index")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + + long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount + newCount)/nodesInIndex : 0; if (dirtyCount + newCount + contentYetToSee > 0) { // We now use the elapsed time as seen by the single thread farming out alc indexing @@ -463,14 +545,14 @@ public class HandlerReportBuilder { srv.getTrackerStats().getNodeTimes().getNamedList(detail, hist, values)); coreSummary.add("Docs/Tx", srv.getTrackerStats().getTxDocs().getNamedList(detail, hist, values)); coreSummary.add("Doc Transformation time (ms)", srv.getTrackerStats().getDocTransformationTimes() - .getNamedList(detail, hist, values)); + .getNamedList(detail, hist, values)); // Model Map> modelErrors = srv.getModelErrors(); if (modelErrors.size() > 0) { - NamedList errorList = new SimpleOrderedMap(); + NamedList errorList = new SimpleOrderedMap<>(); for (Map.Entry> modelNameToErrors : modelErrors.entrySet()) { errorList.add(modelNameToErrors.getKey(), modelNameToErrors.getValue()); @@ -481,4 +563,4 @@ public class HandlerReportBuilder { report.add(cname, coreSummary); } -} +} \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java index 0b41cba15..f8945ef94 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java @@ -35,7 +35,7 @@ import org.alfresco.solr.tracker.CommitTracker; import org.alfresco.solr.tracker.ContentTracker; import org.alfresco.solr.tracker.MetadataTracker; import org.alfresco.solr.tracker.ModelTracker; -import org.alfresco.solr.tracker.SlaveNodeStatePublisher; +import org.alfresco.solr.tracker.SlaveCoreStatePublisher; import org.alfresco.solr.tracker.SolrTrackerScheduler; import org.alfresco.solr.tracker.Tracker; import org.alfresco.solr.tracker.TrackerRegistry; @@ -182,7 +182,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener { LOGGER.info("SearchServices Core Trackers have been explicitly disabled on core \"{}\" through \"enable.alfresco.tracking\" configuration property.", core.getName()); - SlaveNodeStatePublisher statePublisher = new SlaveNodeStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); + SlaveCoreStatePublisher statePublisher = new SlaveCoreStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); trackerRegistry.register(core.getName(), statePublisher); scheduler.schedule(statePublisher, core.getName(), coreProperties); trackers.add(statePublisher); @@ -197,7 +197,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener { LOGGER.info("SearchServices Core Trackers have been disabled on core \"{}\" because it is a slave core.", core.getName()); - SlaveNodeStatePublisher statePublisher = new SlaveNodeStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); + SlaveCoreStatePublisher statePublisher = new SlaveCoreStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); trackerRegistry.register(core.getName(), statePublisher); scheduler.schedule(statePublisher, core.getName(), coreProperties); trackers.add(statePublisher); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/NodeStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java similarity index 91% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/NodeStatePublisher.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java index d6fd2d78a..6c38fb47a 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/NodeStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java @@ -41,6 +41,7 @@ import org.alfresco.service.namespace.QName; import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.InformationServer; +import org.alfresco.solr.NodeReport; import org.alfresco.solr.TrackerState; import org.alfresco.solr.client.SOLRAPIClient; import org.apache.commons.lang3.StringUtils; @@ -59,7 +60,7 @@ import java.util.Properties; * @since 1.5 * @see SEARCH-1752 */ -public abstract class NodeStatePublisher extends AbstractTracker +public abstract class CoreStatePublisher extends AbstractTracker { DocRouter docRouter; private final boolean isMaster; @@ -70,7 +71,7 @@ public abstract class NodeStatePublisher extends AbstractTracker /** The property to use for determining the shard. */ protected Optional shardProperty = Optional.empty(); - NodeStatePublisher( + CoreStatePublisher( boolean isMaster, Properties p, SOLRAPIClient client, @@ -88,12 +89,28 @@ public abstract class NodeStatePublisher extends AbstractTracker docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod)); } - NodeStatePublisher(Type type) + CoreStatePublisher(Type type) { super(type); this.isMaster = false; } + /** + * Returns information about the {@link org.alfresco.solr.client.Node} associated with the given dbid. + * + * @param dbid the node identifier. + * @return the {@link org.alfresco.solr.client.Node} associated with the given dbid. + */ + public NodeReport checkNode(Long dbid) + { + NodeReport nodeReport = new NodeReport(); + nodeReport.setDbid(dbid); + + this.infoSrv.addCommonNodeReportInfo(nodeReport); + + return nodeReport; + } + private void firstUpdateShardProperty() { shardKey.ifPresent( shardKeyName -> { @@ -222,4 +239,14 @@ public abstract class NodeStatePublisher extends AbstractTracker { return this.docRouter; } + + /** + * Returns true if the hosting core is master or standalone. + * + * @return true if the hosting core is master or standalone. + */ + public boolean isOnMasterOrStandalone() + { + return isMaster; + } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index f118c523b..3862ebd30 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -49,7 +49,7 @@ import org.slf4j.LoggerFactory; * This tracks two things: transactions and metadata nodes * @author Ahmed Owian */ -public class MetadataTracker extends NodeStatePublisher implements Tracker +public class MetadataTracker extends CoreStatePublisher implements Tracker { protected final static Logger log = LoggerFactory.getLogger(MetadataTracker.class); private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 100; @@ -888,24 +888,18 @@ public class MetadataTracker extends NodeStatePublisher implements Tracker } } - - - - + @Override public NodeReport checkNode(Long dbid) { - NodeReport nodeReport = new NodeReport(); - nodeReport.setDbid(dbid); + NodeReport nodeReport = super.checkNode(dbid); // In DB - GetNodesParameters parameters = new GetNodesParameters(); parameters.setFromNodeId(dbid); parameters.setToNodeId(dbid); - List dbnodes; try { - dbnodes = client.getNodes(parameters, 1); + List dbnodes = client.getNodes(parameters, 1); if (dbnodes.size() == 1) { Node dbnode = dbnodes.get(0); @@ -915,41 +909,31 @@ public class MetadataTracker extends NodeStatePublisher implements Tracker else { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); - nodeReport.setDbTx(-1l); + nodeReport.setDbTx(-1L); } } catch (IOException e) { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); - nodeReport.setDbTx(-2l); + nodeReport.setDbTx(-2L); } catch (JSONException e) { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); - nodeReport.setDbTx(-3l); + nodeReport.setDbTx(-3L); } catch (AuthenticationException e1) { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); - nodeReport.setDbTx(-4l); + nodeReport.setDbTx(-4L); } - - this.infoSrv.addCommonNodeReportInfo(nodeReport); return nodeReport; } public NodeReport checkNode(Node node) { - NodeReport nodeReport = new NodeReport(); - nodeReport.setDbid(node.getId()); - - nodeReport.setDbNodeStatus(node.getStatus()); - nodeReport.setDbTx(node.getTxnId()); - - this.infoSrv.addCommonNodeReportInfo(nodeReport); - - return nodeReport; + return checkNode(node.getId()); } public List getFullNodesForDbTransaction(Long txid) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveNodeStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java similarity index 90% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveNodeStatePublisher.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java index 4e7ccf709..19a93631c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveNodeStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java @@ -14,8 +14,8 @@ import java.util.Properties; /** * Despite belonging to the Tracker ecosystem, this component is actually a publisher, which periodically informs - * Alfresco about the state of the hosting slave node. - * As the name suggests, this worker is scheduled only when the hosting node acts as a slave. + * Alfresco about the state of the hosting slave core. + * As the name suggests, this worker is scheduled only when the owning core acts as a slave. * It allows Solr's master/slave setup to be used with dynamic shard registration. * * In this scenario the slave is polling a "tracking" Solr node. The tracker below calls @@ -27,9 +27,9 @@ import java.util.Properties; * @author Andrea Gazzarini * @since 1.5 */ -public class SlaveNodeStatePublisher extends NodeStatePublisher +public class SlaveCoreStatePublisher extends CoreStatePublisher { - public SlaveNodeStatePublisher( + public SlaveCoreStatePublisher( boolean isMaster, Properties coreProperties, SOLRAPIClient repositoryClient, @@ -61,6 +61,12 @@ public class SlaveNodeStatePublisher extends NodeStatePublisher // Do nothing here } + @Override + public boolean isOnMasterOrStandalone() + { + return false; + } + @Override public boolean hasMaintenance() { From 9d0eae0f5e60248a81ae8cbd9d66057d0c49baf2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2019 22:14:16 +0000 Subject: [PATCH 61/76] Bump utility from 3.0.13 to 3.0.14 in /e2e-test Bumps [utility](https://github.com/Alfresco/alfresco-tas-utility) from 3.0.13 to 3.0.14. - [Release notes](https://github.com/Alfresco/alfresco-tas-utility/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-utility/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-utility/compare/utility-3.0.13...utility-3.0.14) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index aa7f12854..5cee7b842 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -13,7 +13,7 @@ 6.0.1.2 6.0.0.4 - 3.0.13 + 3.0.14 3.2.0 src/test/resources/SearchSuite.xml From 4f3be9dd06b557aa66977ea56a2f2e24516d44a6 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 6 Nov 2019 11:43:31 +0100 Subject: [PATCH 62/76] [ SEARCH-1917 ] Class comment --- .../org/alfresco/solr/AlfrescoCoreAdminHandler.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index 9badb9506..b8e5c36b2 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -89,6 +89,10 @@ import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Alfresco Solr administration endpoints provider. + * A customisation of the existing Solr {@link CoreAdminHandler} which offers additional administration endpoints. + */ public class AlfrescoCoreAdminHandler extends CoreAdminHandler { protected static final Logger LOGGER = LoggerFactory.getLogger(AlfrescoCoreAdminHandler.class); @@ -124,14 +128,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler public AlfrescoCoreAdminHandler(CoreContainer coreContainer) { super(coreContainer); - startup(coreContainer); - } - /** - * Startup services that exist outside of the core. - */ - public void startup(CoreContainer coreContainer) - { LOGGER.info("Starting Alfresco Core Administration Services"); trackerRegistry = new TrackerRegistry(); From c5c81110c360c37016dcc185c198434540390df3 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 7 Nov 2019 09:22:41 +0100 Subject: [PATCH 63/76] [contentStoreReplication] fix classpath of replication handler in test config files --- .../src/test/resources/test-files/master/conf/solrconfig.xml | 2 -- .../test-files/solrconfig_empty_replication_handler.xml | 2 +- .../solrconfig_master_disabled_replication_handler.xml | 2 +- .../test-files/solrconfig_master_replication_handler.xml | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml index cd99467e1..c05053d6e 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml @@ -188,12 +188,10 @@ - commit schema.xml - /Users/agazzarini/workspaces/alfresco/spike-custom-replication-handler/src/test/resources/contentstore diff --git a/search-services/alfresco-search/src/test/resources/test-files/solrconfig_empty_replication_handler.xml b/search-services/alfresco-search/src/test/resources/test-files/solrconfig_empty_replication_handler.xml index e05ab0538..1c830555f 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/solrconfig_empty_replication_handler.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/solrconfig_empty_replication_handler.xml @@ -15,7 +15,7 @@ - + commit schema.xml diff --git a/search-services/alfresco-search/src/test/resources/test-files/solrconfig_master_disabled_replication_handler.xml b/search-services/alfresco-search/src/test/resources/test-files/solrconfig_master_disabled_replication_handler.xml index 0ed005b25..682674962 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/solrconfig_master_disabled_replication_handler.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/solrconfig_master_disabled_replication_handler.xml @@ -15,7 +15,7 @@ - + false commit diff --git a/search-services/alfresco-search/src/test/resources/test-files/solrconfig_master_replication_handler.xml b/search-services/alfresco-search/src/test/resources/test-files/solrconfig_master_replication_handler.xml index e05ab0538..1c830555f 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/solrconfig_master_replication_handler.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/solrconfig_master_replication_handler.xml @@ -15,7 +15,7 @@ - + commit schema.xml From 4deb377fc4e6a920f8ce4ed44ae7b0a656826cf4 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 7 Nov 2019 11:42:16 +0100 Subject: [PATCH 64/76] [contentStoreReplication] deleted old test --- .../handler/contentStoreReplicationTest.java | 267 ------------------ 1 file changed, 267 deletions(-) delete mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java deleted file mode 100644 index fd4c1e24f..000000000 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/contentStoreReplicationTest.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright (C) 2005-2019 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.handler; - -import java.io.IOException; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; -import org.alfresco.solr.client.Acl; -import org.alfresco.solr.client.AclChangeSet; -import org.alfresco.solr.client.AclReaders; -import org.alfresco.solr.client.Node; -import org.alfresco.solr.client.NodeMetaData; -import org.alfresco.solr.client.Transaction; -import org.apache.commons.io.FileUtils; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.TermQuery; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Properties; - -import static java.util.Collections.singletonList; -import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; -import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; -import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; -import static org.alfresco.solr.AlfrescoSolrUtils.getNode; -import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; -import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; -import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; -import static org.alfresco.solr.utils.AlfrescoFileUtils.areDirectoryEquals; -import static org.carrot2.shaded.guava.common.collect.ImmutableList.of; - -/** - * @author Elia Porciani - * - * This test check if the synchronization of contentstore between master and slave is done correctly. - */ -public class contentStoreReplicationTest extends AbstractAlfrescoDistributedTest { - - protected static JettySolrRunner master; - protected static JettySolrRunner slave; - protected static SolrClient masterClient; - protected static SolrClient slaveClient; - - protected static Path masterSolrHome; - protected static Path slaveSolrHome; - - protected static Path masterContentStore; - protected static Path slaveContentStore; - - private static Acl acl; - - private static final int MILLIS_TIMOUT = 80000; - - @BeforeClass - public static void createMasterSlaveEnv() throws Exception - { - Properties properties = new Properties(); - - clientShards = new ArrayList<>(); - solrShards = new ArrayList<>(); - solrCollectionNameToStandaloneClient = new HashMap<>(); - jettyContainers = new HashMap<>(); - - String coreName = "master"; - boolean basicAuth = properties != null ? Boolean.parseBoolean(properties.getProperty("BasicAuth", "false")) : false; - - String masterKey = "master/solrHome"; - String slaveKey = "slave/solrHome"; - - master = createJetty(masterKey, basicAuth); - addCoreToJetty(masterKey, coreName, coreName, null); - startJetty(master); - - - String slaveCoreName = "slave"; - slave = createJetty(slaveKey, basicAuth); - addCoreToJetty(slaveKey, slaveCoreName, slaveCoreName, null); - setMasterUrl(slaveKey, slaveCoreName, master.getBaseUrl().toString() + "/master"); - - startJetty(slave); - - String masterStr = buildUrl(master.getLocalPort()) + "/" + coreName; - String slaveStr = buildUrl(slave.getLocalPort()) + "/" + slaveCoreName; - - masterClient = createNewSolrClient(masterStr); - slaveClient = createNewSolrClient(slaveStr); - - masterSolrHome = testDir.toPath().resolve(masterKey); - slaveSolrHome = testDir.toPath().resolve(slaveKey); - masterContentStore = testDir.toPath().resolve("master/contentstore"); - slaveContentStore = testDir.toPath().resolve("slave/contentstore"); - - AclChangeSet aclChangeSet = getAclChangeSet(1); - - acl = getAcl(aclChangeSet); - AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); - - indexAclChangeSet(aclChangeSet, - of(acl), - of(aclReaders)); - } - - - @AfterClass - public static void cleanupMasterSlave() throws Exception { - master.stop(); - slave.stop(); - FileUtils.forceDelete(new File(masterSolrHome.getParent().toUri())); - FileUtils.forceDelete(new File(slaveSolrHome.getParent().toUri())); - } - - - @Test - public void contentStoreReplicationTest() throws Exception { - // ADD 250 nodes and check they are replicated - int numNodes = 250; - Transaction bigTxn = getTransaction(0, numNodes); - List nodes = new ArrayList<>(); - List nodeMetaDatas = new ArrayList<>(); - for(int i = 0; i updateNodes = new ArrayList<>(); - List updateNodeMetaDatas = new ArrayList<>(); - for(int i = numNodes; i < totalNodes; i++) { - Node node = getNode(i, updateTx, acl, Node.SolrApiNodeStatus.UPDATED); - updateNodes.add(node); - NodeMetaData nodeMetaData = getNodeMetaData(node, updateTx, acl, "mike", null, false); - node.setNodeRef(nodeMetaData.getNodeRef().toString()); - updateNodeMetaDatas.add(nodeMetaData); - } - - indexTransaction(updateTx, updateNodes, updateNodeMetaDatas); - - waitForDocCountCore(masterClient, - luceneToSolrQuery(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world"))), - numNodes + numUpdates, MILLIS_TIMOUT, System.currentTimeMillis()); - - filesInMasterContentStore = Files.walk(Paths.get(masterContentStore.toUri().resolve("_DEFAULT_"))) - .filter(Files::isRegularFile) - .count(); - - Assert.assertEquals( "master contentStore should have " + totalNodes + "files", totalNodes, filesInMasterContentStore); - assertTrue("slave content store is not in sync after timeout", waitForContentStoreSync(MILLIS_TIMOUT)); - - - // DELETES 30 nodes - int numDeletes = 30; - Transaction deleteTx = getTransaction(numDeletes, 0); - - totalNodes = numNodes + numUpdates - numDeletes; - List deleteNodes = new ArrayList<>(); - List deleteNodeMetaDatas = new ArrayList<>(); - - for(int i = 0; i Date: Thu, 7 Nov 2019 12:02:08 +0100 Subject: [PATCH 65/76] [contentStoreReplication] Fix class name after merging from master --- .../alfresco/solr/handler/ContentStoreReplicationIT.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java index a1a7fb206..d903b3974 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java @@ -18,8 +18,7 @@ */ package org.alfresco.solr.handler; -import java.io.IOException; -import org.alfresco.solr.AbstractAlfrescoDistributedTest; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; @@ -38,6 +37,7 @@ import org.junit.BeforeClass; import org.junit.Test; import java.io.File; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -64,7 +64,8 @@ import static org.carrot2.shaded.guava.common.collect.ImmutableList.of; * This test check if the synchronization of contentstore between master and slave is done correctly. */ @SolrTestCaseJ4.SuppressSSL -public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedTest { +public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT +{ protected static JettySolrRunner master; protected static JettySolrRunner slave; From 9f3ec7bebf9b31e64c8a1c1dc31ae20f5186c9fa Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 7 Nov 2019 16:16:26 +0100 Subject: [PATCH 66/76] [contentStoreReplication] Restored solrhome property in test initialization --- .../src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java index 52fc281a1..299cb351d 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java @@ -242,6 +242,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS LOG.info("##################################### init Alfresco core ##############"); LOG.info("####initCore"); + System.setProperty("solr.solr.home", TEST_FILES_LOCATION); System.setProperty("solr.directoryFactory","solr.RAMDirectoryFactory"); System.setProperty("solr.tests.maxBufferedDocs", "1000"); System.setProperty("solr.tests.maxIndexingThreads", "10"); From e50c4a3ea8661966ae71bbb3d60ea9b76c01603c Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 7 Nov 2019 17:03:58 +0100 Subject: [PATCH 67/76] [contentStoreReplication] Removed unnecessary files for contentStoreReplicationTest --- search-services/alfresco-search/pom.xml | 39 + .../test-files/master/conf/admin-extra.html | 156 ---- .../master/conf/admin-extra.menu-bottom.html | 0 .../master/conf/admin-extra.menu-top.html | 0 .../test-files/master/conf/elevate.xml | 38 - .../master/conf/lang/contractions_ca.txt | 8 - .../master/conf/lang/contractions_fr.txt | 15 - .../master/conf/lang/contractions_ga.txt | 5 - .../master/conf/lang/contractions_it.txt | 23 - .../master/conf/lang/hyphenations_ga.txt | 5 - .../master/conf/lang/stemdict_nl.txt | 6 - .../master/conf/lang/stoptags_ja.txt | 420 ---------- .../master/conf/lang/stopwords_ar.txt | 125 --- .../master/conf/lang/stopwords_bg.txt | 193 ----- .../master/conf/lang/stopwords_ca.txt | 220 ----- .../master/conf/lang/stopwords_ckb.txt | 136 ---- .../master/conf/lang/stopwords_cz.txt | 172 ---- .../master/conf/lang/stopwords_da.txt | 110 --- .../master/conf/lang/stopwords_de.txt | 294 ------- .../master/conf/lang/stopwords_el.txt | 78 -- .../master/conf/lang/stopwords_en.txt | 332 -------- .../master/conf/lang/stopwords_es.txt | 356 -------- .../master/conf/lang/stopwords_eu.txt | 99 --- .../master/conf/lang/stopwords_fa.txt | 313 ------- .../master/conf/lang/stopwords_fi.txt | 97 --- .../master/conf/lang/stopwords_fr.txt | 186 ----- .../master/conf/lang/stopwords_ga.txt | 110 --- .../master/conf/lang/stopwords_gl.txt | 161 ---- .../master/conf/lang/stopwords_hi.txt | 235 ------ .../master/conf/lang/stopwords_hu.txt | 211 ----- .../master/conf/lang/stopwords_hy.txt | 46 -- .../master/conf/lang/stopwords_id.txt | 359 -------- .../master/conf/lang/stopwords_it.txt | 303 ------- .../master/conf/lang/stopwords_ja.txt | 127 --- .../master/conf/lang/stopwords_lv.txt | 172 ---- .../master/conf/lang/stopwords_nl.txt | 119 --- .../master/conf/lang/stopwords_no.txt | 194 ----- .../master/conf/lang/stopwords_pt.txt | 253 ------ .../master/conf/lang/stopwords_ro.txt | 233 ------ .../master/conf/lang/stopwords_ru.txt | 243 ------ .../master/conf/lang/stopwords_sv.txt | 133 --- .../master/conf/lang/stopwords_th.txt | 119 --- .../master/conf/lang/stopwords_tr.txt | 212 ----- .../master/conf/lang/userdict_ja.txt | 29 - .../test-files/master/conf/protwords.txt | 21 - .../test-files/master/conf/schema.xml | 766 ------------------ .../test-files/master/conf/spellings.txt | 2 - .../test-files/slave/conf/admin-extra.html | 156 ---- .../slave/conf/admin-extra.menu-bottom.html | 0 .../slave/conf/admin-extra.menu-top.html | 0 .../test-files/slave/conf/elevate.xml | 38 - .../slave/conf/lang/contractions_ca.txt | 8 - .../slave/conf/lang/contractions_fr.txt | 15 - .../slave/conf/lang/contractions_ga.txt | 5 - .../slave/conf/lang/contractions_it.txt | 23 - .../slave/conf/lang/hyphenations_ga.txt | 5 - .../slave/conf/lang/stemdict_nl.txt | 6 - .../slave/conf/lang/stoptags_ja.txt | 420 ---------- .../slave/conf/lang/stopwords_ar.txt | 125 --- .../slave/conf/lang/stopwords_bg.txt | 193 ----- .../slave/conf/lang/stopwords_ca.txt | 220 ----- .../slave/conf/lang/stopwords_ckb.txt | 136 ---- .../slave/conf/lang/stopwords_cz.txt | 172 ---- .../slave/conf/lang/stopwords_da.txt | 110 --- .../slave/conf/lang/stopwords_de.txt | 294 ------- .../slave/conf/lang/stopwords_el.txt | 78 -- .../slave/conf/lang/stopwords_en.txt | 332 -------- .../slave/conf/lang/stopwords_es.txt | 356 -------- .../slave/conf/lang/stopwords_eu.txt | 99 --- .../slave/conf/lang/stopwords_fa.txt | 313 ------- .../slave/conf/lang/stopwords_fi.txt | 97 --- .../slave/conf/lang/stopwords_fr.txt | 186 ----- .../slave/conf/lang/stopwords_ga.txt | 110 --- .../slave/conf/lang/stopwords_gl.txt | 161 ---- .../slave/conf/lang/stopwords_hi.txt | 235 ------ .../slave/conf/lang/stopwords_hu.txt | 211 ----- .../slave/conf/lang/stopwords_hy.txt | 46 -- .../slave/conf/lang/stopwords_id.txt | 359 -------- .../slave/conf/lang/stopwords_it.txt | 303 ------- .../slave/conf/lang/stopwords_ja.txt | 127 --- .../slave/conf/lang/stopwords_lv.txt | 172 ---- .../slave/conf/lang/stopwords_nl.txt | 119 --- .../slave/conf/lang/stopwords_no.txt | 194 ----- .../slave/conf/lang/stopwords_pt.txt | 253 ------ .../slave/conf/lang/stopwords_ro.txt | 233 ------ .../slave/conf/lang/stopwords_ru.txt | 243 ------ .../slave/conf/lang/stopwords_sv.txt | 133 --- .../slave/conf/lang/stopwords_th.txt | 119 --- .../slave/conf/lang/stopwords_tr.txt | 212 ----- .../slave/conf/lang/userdict_ja.txt | 29 - .../test-files/slave/conf/protwords.txt | 21 - .../test-files/slave/conf/schema.xml | 766 ------------------ .../test-files/slave/conf/spellings.txt | 2 - 93 files changed, 39 insertions(+), 14870 deletions(-) delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/admin-extra.html delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/admin-extra.menu-bottom.html delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/admin-extra.menu-top.html delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/elevate.xml delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ca.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_fr.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ga.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_it.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/hyphenations_ga.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stemdict_nl.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stoptags_ja.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ar.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_bg.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ca.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ckb.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_cz.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_da.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_de.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_el.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_en.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_es.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_eu.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fa.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fi.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fr.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ga.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_gl.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hi.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hu.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hy.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_id.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_it.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ja.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_lv.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_nl.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_no.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_pt.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ro.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ru.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_sv.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_th.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_tr.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/userdict_ja.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/protwords.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/schema.xml delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/master/conf/spellings.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.html delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.menu-bottom.html delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.menu-top.html delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/elevate.xml delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ca.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_fr.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ga.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_it.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/hyphenations_ga.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stemdict_nl.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stoptags_ja.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ar.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_bg.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ca.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ckb.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_cz.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_da.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_de.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_el.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_en.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_es.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_eu.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fa.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fi.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fr.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ga.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_gl.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hi.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hu.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hy.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_id.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_it.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ja.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_lv.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_nl.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_no.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_pt.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ro.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ru.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_sv.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_th.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_tr.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/userdict_ja.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/protwords.txt delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/schema.xml delete mode 100644 search-services/alfresco-search/src/test/resources/test-files/slave/conf/spellings.txt diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index f5285a7cf..8cdd7d1f7 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -179,6 +179,45 @@ src/main/resources/solr/instance/templates/rerank/conf solrconfig.xml + solrcore.properties + + + + + + + copy-production-solr-configuration-for-master + generate-test-resources + + copy-resources + + + ${project.build.testOutputDirectory}/test-files/master/conf + + + src/main/resources/solr/instance/templates/rerank/conf + + solrconfig.xml + solrcore.properties + + + + + + + copy-production-solr-configuration-for-slave + generate-test-resources + + copy-resources + + + ${project.build.testOutputDirectory}/test-files/slave/conf + + + src/main/resources/solr/instance/templates/rerank/conf + + solrconfig.xml + solrcore.properties diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/admin-extra.html b/search-services/alfresco-search/src/test/resources/test-files/master/conf/admin-extra.html deleted file mode 100644 index d8f22b44e..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/admin-extra.html +++ /dev/null @@ -1,156 +0,0 @@ - - - - - -
    Update the Summary and FTS Status reports
    - -
    -
    -

    Alfresco Core - Summary Report

    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -

    Alfresco Core - FTS Status Report

    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -
    - - - - -

    Other Links:

    -
    -
    Note: the following links in a new window
    - - - -
    diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/admin-extra.menu-bottom.html b/search-services/alfresco-search/src/test/resources/test-files/master/conf/admin-extra.menu-bottom.html deleted file mode 100644 index e69de29bb..000000000 diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/admin-extra.menu-top.html b/search-services/alfresco-search/src/test/resources/test-files/master/conf/admin-extra.menu-top.html deleted file mode 100644 index e69de29bb..000000000 diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/elevate.xml b/search-services/alfresco-search/src/test/resources/test-files/master/conf/elevate.xml deleted file mode 100644 index ed2bc4c3a..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/elevate.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ca.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ca.txt deleted file mode 100644 index 307a85f91..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ca.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Set of Catalan contractions for ElisionFilter -# TODO: load this as a resource from the analyzer and sync it in build.xml -d -l -m -n -s -t diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_fr.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_fr.txt deleted file mode 100644 index f1bba51b2..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_fr.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Set of French contractions for ElisionFilter -# TODO: load this as a resource from the analyzer and sync it in build.xml -l -m -t -qu -n -s -j -d -c -jusqu -quoiqu -lorsqu -puisqu diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ga.txt deleted file mode 100644 index 9ebe7fa34..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_ga.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Set of Irish contractions for ElisionFilter -# TODO: load this as a resource from the analyzer and sync it in build.xml -d -m -b diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_it.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_it.txt deleted file mode 100644 index cac040953..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/contractions_it.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Set of Italian contractions for ElisionFilter -# TODO: load this as a resource from the analyzer and sync it in build.xml -c -l -all -dall -dell -nell -sull -coll -pell -gl -agl -dagl -degl -negl -sugl -un -m -t -s -v -d diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/hyphenations_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/hyphenations_ga.txt deleted file mode 100644 index 4d2642cc5..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/hyphenations_ga.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Set of Irish hyphenations for StopFilter -# TODO: load this as a resource from the analyzer and sync it in build.xml -h -n -t diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stemdict_nl.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stemdict_nl.txt deleted file mode 100644 index 441072971..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stemdict_nl.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Set of overrides for the dutch stemmer -# TODO: load this as a resource from the analyzer and sync it in build.xml -fiets fiets -bromfiets bromfiets -ei eier -kind kinder diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stoptags_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stoptags_ja.txt deleted file mode 100644 index 71b750845..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stoptags_ja.txt +++ /dev/null @@ -1,420 +0,0 @@ -# -# This file defines a Japanese stoptag set for JapanesePartOfSpeechStopFilter. -# -# Any token with a part-of-speech tag that exactly matches those defined in this -# file are removed from the token stream. -# -# Set your own stoptags by uncommenting the lines below. Note that comments are -# not allowed on the same line as a stoptag. See LUCENE-3745 for frequency lists, -# etc. that can be useful for building you own stoptag set. -# -# The entire possible tagset is provided below for convenience. -# -##### -# noun: unclassified nouns -#名詞 -# -# noun-common: Common nouns or nouns where the sub-classification is undefined -#名詞-一般 -# -# noun-proper: Proper nouns where the sub-classification is undefined -#名詞-固有名詞 -# -# noun-proper-misc: miscellaneous proper nouns -#名詞-固有名詞-一般 -# -# noun-proper-person: Personal names where the sub-classification is undefined -#名詞-固有名詞-人名 -# -# noun-proper-person-misc: names that cannot be divided into surname and -# given name; foreign names; names where the surname or given name is unknown. -# e.g. お市の方 -#名詞-固有名詞-人名-一般 -# -# noun-proper-person-surname: Mainly Japanese surnames. -# e.g. 山田 -#名詞-固有名詞-人名-姓 -# -# noun-proper-person-given_name: Mainly Japanese given names. -# e.g. 太郎 -#名詞-固有名詞-人名-名 -# -# noun-proper-organization: Names representing organizations. -# e.g. 通産省, NHK -#名詞-固有名詞-組織 -# -# noun-proper-place: Place names where the sub-classification is undefined -#名詞-固有名詞-地域 -# -# noun-proper-place-misc: Place names excluding countries. -# e.g. アジア, バルセロナ, 京都 -#名詞-固有名詞-地域-一般 -# -# noun-proper-place-country: Country names. -# e.g. 日本, オーストラリア -#名詞-固有名詞-地域-国 -# -# noun-pronoun: Pronouns where the sub-classification is undefined -#名詞-代名詞 -# -# noun-pronoun-misc: miscellaneous pronouns: -# e.g. それ, ここ, あいつ, あなた, あちこち, いくつ, どこか, なに, みなさん, みんな, わたくし, われわれ -#名詞-代名詞-一般 -# -# noun-pronoun-contraction: Spoken language contraction made by combining a -# pronoun and the particle 'wa'. -# e.g. ありゃ, こりゃ, こりゃあ, そりゃ, そりゃあ -#名詞-代名詞-縮約 -# -# noun-adverbial: Temporal nouns such as names of days or months that behave -# like adverbs. Nouns that represent amount or ratios and can be used adverbially, -# e.g. 金曜, 一月, 午後, 少量 -#名詞-副詞可能 -# -# noun-verbal: Nouns that take arguments with case and can appear followed by -# 'suru' and related verbs (する, できる, なさる, くださる) -# e.g. インプット, 愛着, 悪化, 悪戦苦闘, 一安心, 下取り -#名詞-サ変接続 -# -# noun-adjective-base: The base form of adjectives, words that appear before な ("na") -# e.g. 健康, 安易, 駄目, だめ -#名詞-形容動詞語幹 -# -# noun-numeric: Arabic numbers, Chinese numerals, and counters like 何 (回), 数. -# e.g. 0, 1, 2, 何, 数, 幾 -#名詞-数 -# -# noun-affix: noun affixes where the sub-classification is undefined -#名詞-非自立 -# -# noun-affix-misc: Of adnominalizers, the case-marker の ("no"), and words that -# attach to the base form of inflectional words, words that cannot be classified -# into any of the other categories below. This category includes indefinite nouns. -# e.g. あかつき, 暁, かい, 甲斐, 気, きらい, 嫌い, くせ, 癖, こと, 事, ごと, 毎, しだい, 次第, -# 順, せい, 所為, ついで, 序で, つもり, 積もり, 点, どころ, の, はず, 筈, はずみ, 弾み, -# 拍子, ふう, ふり, 振り, ほう, 方, 旨, もの, 物, 者, ゆえ, 故, ゆえん, 所以, わけ, 訳, -# わり, 割り, 割, ん-口語/, もん-口語/ -#名詞-非自立-一般 -# -# noun-affix-adverbial: noun affixes that that can behave as adverbs. -# e.g. あいだ, 間, あげく, 挙げ句, あと, 後, 余り, 以外, 以降, 以後, 以上, 以前, 一方, うえ, -# 上, うち, 内, おり, 折り, かぎり, 限り, きり, っきり, 結果, ころ, 頃, さい, 際, 最中, さなか, -# 最中, じたい, 自体, たび, 度, ため, 為, つど, 都度, とおり, 通り, とき, 時, ところ, 所, -# とたん, 途端, なか, 中, のち, 後, ばあい, 場合, 日, ぶん, 分, ほか, 他, まえ, 前, まま, -# 儘, 侭, みぎり, 矢先 -#名詞-非自立-副詞可能 -# -# noun-affix-aux: noun affixes treated as 助動詞 ("auxiliary verb") in school grammars -# with the stem よう(だ) ("you(da)"). -# e.g. よう, やう, 様 (よう) -#名詞-非自立-助動詞語幹 -# -# noun-affix-adjective-base: noun affixes that can connect to the indeclinable -# connection form な (aux "da"). -# e.g. みたい, ふう -#名詞-非自立-形容動詞語幹 -# -# noun-special: special nouns where the sub-classification is undefined. -#名詞-特殊 -# -# noun-special-aux: The そうだ ("souda") stem form that is used for reporting news, is -# treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the base -# form of inflectional words. -# e.g. そう -#名詞-特殊-助動詞語幹 -# -# noun-suffix: noun suffixes where the sub-classification is undefined. -#名詞-接尾 -# -# noun-suffix-misc: Of the nouns or stem forms of other parts of speech that connect -# to ガル or タイ and can combine into compound nouns, words that cannot be classified into -# any of the other categories below. In general, this category is more inclusive than -# 接尾語 ("suffix") and is usually the last element in a compound noun. -# e.g. おき, かた, 方, 甲斐 (がい), がかり, ぎみ, 気味, ぐるみ, (～した) さ, 次第, 済 (ず) み, -# よう, (でき)っこ, 感, 観, 性, 学, 類, 面, 用 -#名詞-接尾-一般 -# -# noun-suffix-person: Suffixes that form nouns and attach to person names more often -# than other nouns. -# e.g. 君, 様, 著 -#名詞-接尾-人名 -# -# noun-suffix-place: Suffixes that form nouns and attach to place names more often -# than other nouns. -# e.g. 町, 市, 県 -#名詞-接尾-地域 -# -# noun-suffix-verbal: Of the suffixes that attach to nouns and form nouns, those that -# can appear before スル ("suru"). -# e.g. 化, 視, 分け, 入り, 落ち, 買い -#名詞-接尾-サ変接続 -# -# noun-suffix-aux: The stem form of そうだ (様態) that is used to indicate conditions, -# is treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the -# conjunctive form of inflectional words. -# e.g. そう -#名詞-接尾-助動詞語幹 -# -# noun-suffix-adjective-base: Suffixes that attach to other nouns or the conjunctive -# form of inflectional words and appear before the copula だ ("da"). -# e.g. 的, げ, がち -#名詞-接尾-形容動詞語幹 -# -# noun-suffix-adverbial: Suffixes that attach to other nouns and can behave as adverbs. -# e.g. 後 (ご), 以後, 以降, 以前, 前後, 中, 末, 上, 時 (じ) -#名詞-接尾-副詞可能 -# -# noun-suffix-classifier: Suffixes that attach to numbers and form nouns. This category -# is more inclusive than 助数詞 ("classifier") and includes common nouns that attach -# to numbers. -# e.g. 個, つ, 本, 冊, パーセント, cm, kg, カ月, か国, 区画, 時間, 時半 -#名詞-接尾-助数詞 -# -# noun-suffix-special: Special suffixes that mainly attach to inflecting words. -# e.g. (楽し) さ, (考え) 方 -#名詞-接尾-特殊 -# -# noun-suffix-conjunctive: Nouns that behave like conjunctions and join two words -# together. -# e.g. (日本) 対 (アメリカ), 対 (アメリカ), (3) 対 (5), (女優) 兼 (主婦) -#名詞-接続詞的 -# -# noun-verbal_aux: Nouns that attach to the conjunctive particle て ("te") and are -# semantically verb-like. -# e.g. ごらん, ご覧, 御覧, 頂戴 -#名詞-動詞非自立的 -# -# noun-quotation: text that cannot be segmented into words, proverbs, Chinese poetry, -# dialects, English, etc. Currently, the only entry for 名詞 引用文字列 ("noun quotation") -# is いわく ("iwaku"). -#名詞-引用文字列 -# -# noun-nai_adjective: Words that appear before the auxiliary verb ない ("nai") and -# behave like an adjective. -# e.g. 申し訳, 仕方, とんでも, 違い -#名詞-ナイ形容詞語幹 -# -##### -# prefix: unclassified prefixes -#接頭詞 -# -# prefix-nominal: Prefixes that attach to nouns (including adjective stem forms) -# excluding numerical expressions. -# e.g. お (水), 某 (氏), 同 (社), 故 (～氏), 高 (品質), お (見事), ご (立派) -#接頭詞-名詞接続 -# -# prefix-verbal: Prefixes that attach to the imperative form of a verb or a verb -# in conjunctive form followed by なる/なさる/くださる. -# e.g. お (読みなさい), お (座り) -#接頭詞-動詞接続 -# -# prefix-adjectival: Prefixes that attach to adjectives. -# e.g. お (寒いですねえ), バカ (でかい) -#接頭詞-形容詞接続 -# -# prefix-numerical: Prefixes that attach to numerical expressions. -# e.g. 約, およそ, 毎時 -#接頭詞-数接続 -# -##### -# verb: unclassified verbs -#動詞 -# -# verb-main: -#動詞-自立 -# -# verb-auxiliary: -#動詞-非自立 -# -# verb-suffix: -#動詞-接尾 -# -##### -# adjective: unclassified adjectives -#形容詞 -# -# adjective-main: -#形容詞-自立 -# -# adjective-auxiliary: -#形容詞-非自立 -# -# adjective-suffix: -#形容詞-接尾 -# -##### -# adverb: unclassified adverbs -#副詞 -# -# adverb-misc: Words that can be segmented into one unit and where adnominal -# modification is not possible. -# e.g. あいかわらず, 多分 -#副詞-一般 -# -# adverb-particle_conjunction: Adverbs that can be followed by の, は, に, -# な, する, だ, etc. -# e.g. こんなに, そんなに, あんなに, なにか, なんでも -#副詞-助詞類接続 -# -##### -# adnominal: Words that only have noun-modifying forms. -# e.g. この, その, あの, どの, いわゆる, なんらかの, 何らかの, いろんな, こういう, そういう, ああいう, -# どういう, こんな, そんな, あんな, どんな, 大きな, 小さな, おかしな, ほんの, たいした, -# 「(, も) さる (ことながら)」, 微々たる, 堂々たる, 単なる, いかなる, 我が」「同じ, 亡き -#連体詞 -# -##### -# conjunction: Conjunctions that can occur independently. -# e.g. が, けれども, そして, じゃあ, それどころか -接続詞 -# -##### -# particle: unclassified particles. -助詞 -# -# particle-case: case particles where the subclassification is undefined. -助詞-格助詞 -# -# particle-case-misc: Case particles. -# e.g. から, が, で, と, に, へ, より, を, の, にて -助詞-格助詞-一般 -# -# particle-case-quote: the "to" that appears after nouns, a person’s speech, -# quotation marks, expressions of decisions from a meeting, reasons, judgements, -# conjectures, etc. -# e.g. ( だ) と (述べた.), ( である) と (して執行猶予...) -助詞-格助詞-引用 -# -# particle-case-compound: Compounds of particles and verbs that mainly behave -# like case particles. -# e.g. という, といった, とかいう, として, とともに, と共に, でもって, にあたって, に当たって, に当って, -# にあたり, に当たり, に当り, に当たる, にあたる, において, に於いて,に於て, における, に於ける, -# にかけ, にかけて, にかんし, に関し, にかんして, に関して, にかんする, に関する, に際し, -# に際して, にしたがい, に従い, に従う, にしたがって, に従って, にたいし, に対し, にたいして, -# に対して, にたいする, に対する, について, につき, につけ, につけて, につれ, につれて, にとって, -# にとり, にまつわる, によって, に依って, に因って, により, に依り, に因り, による, に依る, に因る, -# にわたって, にわたる, をもって, を以って, を通じ, を通じて, を通して, をめぐって, をめぐり, をめぐる, -# って-口語/, ちゅう-関西弁「という」/, (何) ていう (人)-口語/, っていう-口語/, といふ, とかいふ -助詞-格助詞-連語 -# -# particle-conjunctive: -# e.g. から, からには, が, けれど, けれども, けど, し, つつ, て, で, と, ところが, どころか, とも, ども, -# ながら, なり, ので, のに, ば, ものの, や ( した), やいなや, (ころん) じゃ(いけない)-口語/, -# (行っ) ちゃ(いけない)-口語/, (言っ) たって (しかたがない)-口語/, (それがなく)ったって (平気)-口語/ -助詞-接続助詞 -# -# particle-dependency: -# e.g. こそ, さえ, しか, すら, は, も, ぞ -助詞-係助詞 -# -# particle-adverbial: -# e.g. がてら, かも, くらい, 位, ぐらい, しも, (学校) じゃ(これが流行っている)-口語/, -# (それ)じゃあ (よくない)-口語/, ずつ, (私) なぞ, など, (私) なり (に), (先生) なんか (大嫌い)-口語/, -# (私) なんぞ, (先生) なんて (大嫌い)-口語/, のみ, だけ, (私) だって-口語/, だに, -# (彼)ったら-口語/, (お茶) でも (いかが), 等 (とう), (今後) とも, ばかり, ばっか-口語/, ばっかり-口語/, -# ほど, 程, まで, 迄, (誰) も (が)([助詞-格助詞] および [助詞-係助詞] の前に位置する「も」) -助詞-副助詞 -# -# particle-interjective: particles with interjective grammatical roles. -# e.g. (松島) や -助詞-間投助詞 -# -# particle-coordinate: -# e.g. と, たり, だの, だり, とか, なり, や, やら -助詞-並立助詞 -# -# particle-final: -# e.g. かい, かしら, さ, ぜ, (だ)っけ-口語/, (とまってる) で-方言/, な, ナ, なあ-口語/, ぞ, ね, ネ, -# ねぇ-口語/, ねえ-口語/, ねん-方言/, の, のう-口語/, や, よ, ヨ, よぉ-口語/, わ, わい-口語/ -助詞-終助詞 -# -# particle-adverbial/conjunctive/final: The particle "ka" when unknown whether it is -# adverbial, conjunctive, or sentence final. For example: -# (a) 「A か B か」. Ex:「(国内で運用する) か,(海外で運用する) か (.)」 -# (b) Inside an adverb phrase. Ex:「(幸いという) か (, 死者はいなかった.)」 -# 「(祈りが届いたせい) か (, 試験に合格した.)」 -# (c) 「かのように」. Ex:「(何もなかった) か (のように振る舞った.)」 -# e.g. か -助詞-副助詞／並立助詞／終助詞 -# -# particle-adnominalizer: The "no" that attaches to nouns and modifies -# non-inflectional words. -助詞-連体化 -# -# particle-adnominalizer: The "ni" and "to" that appear following nouns and adverbs -# that are giongo, giseigo, or gitaigo. -# e.g. に, と -助詞-副詞化 -# -# particle-special: A particle that does not fit into one of the above classifications. -# This includes particles that are used in Tanka, Haiku, and other poetry. -# e.g. かな, けむ, ( しただろう) に, (あんた) にゃ(わからん), (俺) ん (家) -助詞-特殊 -# -##### -# auxiliary-verb: -助動詞 -# -##### -# interjection: Greetings and other exclamations. -# e.g. おはよう, おはようございます, こんにちは, こんばんは, ありがとう, どうもありがとう, ありがとうございます, -# いただきます, ごちそうさま, さよなら, さようなら, はい, いいえ, ごめん, ごめんなさい -#感動詞 -# -##### -# symbol: unclassified Symbols. -記号 -# -# symbol-misc: A general symbol not in one of the categories below. -# e.g. [○◎@$〒→+] -記号-一般 -# -# symbol-comma: Commas -# e.g. [,、] -記号-読点 -# -# symbol-period: Periods and full stops. -# e.g. [.．。] -記号-句点 -# -# symbol-space: Full-width whitespace. -記号-空白 -# -# symbol-open_bracket: -# e.g. [({‘“『【] -記号-括弧開 -# -# symbol-close_bracket: -# e.g. [)}’”』」】] -記号-括弧閉 -# -# symbol-alphabetic: -#記号-アルファベット -# -##### -# other: unclassified other -#その他 -# -# other-interjection: Words that are hard to classify as noun-suffixes or -# sentence-final particles. -# e.g. (だ)ァ -その他-間投 -# -##### -# filler: Aizuchi that occurs during a conversation or sounds inserted as filler. -# e.g. あの, うんと, えと -フィラー -# -##### -# non-verbal: non-verbal sound. -非言語音 -# -##### -# fragment: -#語断片 -# -##### -# unknown: unknown part of speech. -#未知語 -# -##### End of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ar.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ar.txt deleted file mode 100644 index 046829db6..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ar.txt +++ /dev/null @@ -1,125 +0,0 @@ -# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -# Cleaned on October 11, 2009 (not normalized, so use before normalization) -# This means that when modifying this list, you might need to add some -# redundant entries, for example containing forms with both أ and ا -من -ومن -منها -منه -في -وفي -فيها -فيه -و -ف -ثم -او -أو -ب -بها -به -ا -أ -اى -اي -أي -أى -لا -ولا -الا -ألا -إلا -لكن -ما -وما -كما -فما -عن -مع -اذا -إذا -ان -أن -إن -انها -أنها -إنها -انه -أنه -إنه -بان -بأن -فان -فأن -وان -وأن -وإن -التى -التي -الذى -الذي -الذين -الى -الي -إلى -إلي -على -عليها -عليه -اما -أما -إما -ايضا -أيضا -كل -وكل -لم -ولم -لن -ولن -هى -هي -هو -وهى -وهي -وهو -فهى -فهي -فهو -انت -أنت -لك -لها -له -هذه -هذا -تلك -ذلك -هناك -كانت -كان -يكون -تكون -وكانت -وكان -غير -بعض -قد -نحو -بين -بينما -منذ -ضمن -حيث -الان -الآن -خلال -بعد -قبل -حتى -عند -عندما -لدى -جميع diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_bg.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_bg.txt deleted file mode 100644 index 1ae4ba2ae..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_bg.txt +++ /dev/null @@ -1,193 +0,0 @@ -# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -а -аз -ако -ала -бе -без -беше -би -бил -била -били -било -близо -бъдат -бъде -бяха -в -вас -ваш -ваша -вероятно -вече -взема -ви -вие -винаги -все -всеки -всички -всичко -всяка -във -въпреки -върху -г -ги -главно -го -д -да -дали -до -докато -докога -дори -досега -доста -е -едва -един -ето -за -зад -заедно -заради -засега -затова -защо -защото -и -из -или -им -има -имат -иска -й -каза -как -каква -какво -както -какъв -като -кога -когато -което -които -кой -който -колко -която -къде -където -към -ли -м -ме -между -мен -ми -мнозина -мога -могат -може -моля -момента -му -н -на -над -назад -най -направи -напред -например -нас -не -него -нея -ни -ние -никой -нито -но -някои -някой -няма -обаче -около -освен -особено -от -отгоре -отново -още -пак -по -повече -повечето -под -поне -поради -после -почти -прави -пред -преди -през -при -пък -първо -с -са -само -се -сега -си -скоро -след -сме -според -сред -срещу -сте -съм -със -също -т -тази -така -такива -такъв -там -твой -те -тези -ти -тн -то -това -тогава -този -той -толкова -точно -трябва -тук -тъй -тя -тях -у -харесва -ч -че -често -чрез -ще -щом -я diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ca.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ca.txt deleted file mode 100644 index 3da65deaf..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ca.txt +++ /dev/null @@ -1,220 +0,0 @@ -# Catalan stopwords from http://github.com/vcl/cue.language (Apache 2 Licensed) -a -abans -ací -ah -així -això -al -als -aleshores -algun -alguna -algunes -alguns -alhora -allà -allí -allò -altra -altre -altres -amb -ambdós -ambdues -apa -aquell -aquella -aquelles -aquells -aquest -aquesta -aquestes -aquests -aquí -baix -cada -cadascú -cadascuna -cadascunes -cadascuns -com -contra -d'un -d'una -d'unes -d'uns -dalt -de -del -dels -des -després -dins -dintre -donat -doncs -durant -e -eh -el -els -em -en -encara -ens -entre -érem -eren -éreu -es -és -esta -està -estàvem -estaven -estàveu -esteu -et -etc -ets -fins -fora -gairebé -ha -han -has -havia -he -hem -heu -hi -ho -i -igual -iguals -ja -l'hi -la -les -li -li'n -llavors -m'he -ma -mal -malgrat -mateix -mateixa -mateixes -mateixos -me -mentre -més -meu -meus -meva -meves -molt -molta -moltes -molts -mon -mons -n'he -n'hi -ne -ni -no -nogensmenys -només -nosaltres -nostra -nostre -nostres -o -oh -oi -on -pas -pel -pels -per -però -perquè -poc -poca -pocs -poques -potser -propi -qual -quals -quan -quant -que -què -quelcom -qui -quin -quina -quines -quins -s'ha -s'han -sa -semblant -semblants -ses -seu -seus -seva -seva -seves -si -sobre -sobretot -sóc -solament -sols -son -són -sons -sota -sou -t'ha -t'han -t'he -ta -tal -també -tampoc -tan -tant -tanta -tantes -teu -teus -teva -teves -ton -tons -tot -tota -totes -tots -un -una -unes -uns -us -va -vaig -vam -van -vas -veu -vosaltres -vostra -vostre -vostres diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ckb.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ckb.txt deleted file mode 100644 index 87abf118f..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ckb.txt +++ /dev/null @@ -1,136 +0,0 @@ -# set of kurdish stopwords -# note these have been normalized with our scheme (e represented with U+06D5, etc) -# constructed from: -# * Fig 5 of "Building A Test Collection For Sorani Kurdish" (Esmaili et al) -# * "Sorani Kurdish: A Reference Grammar with selected readings" (Thackston) -# * Corpus-based analysis of 77M word Sorani collection: wikipedia, news, blogs, etc - -# and -و -# which -کە -# of -ی -# made/did -کرد -# that/which -ئەوەی -# on/head -سەر -# two -دوو -# also -هەروەها -# from/that -لەو -# makes/does -دەکات -# some -چەند -# every -هەر - -# demonstratives -# that -ئەو -# this -ئەم - -# personal pronouns -# I -من -# we -ئێمە -# you -تۆ -# you -ئێوە -# he/she/it -ئەو -# they -ئەوان - -# prepositions -# to/with/by -بە -پێ -# without -بەبێ -# along with/while/during -بەدەم -# in the opinion of -بەلای -# according to -بەپێی -# before -بەرلە -# in the direction of -بەرەوی -# in front of/toward -بەرەوە -# before/in the face of -بەردەم -# without -بێ -# except for -بێجگە -# for -بۆ -# on/in -دە -تێ -# with -دەگەڵ -# after -دوای -# except for/aside from -جگە -# in/from -لە -لێ -# in front of/before/because of -لەبەر -# between/among -لەبەینی -# concerning/about -لەبابەت -# concerning -لەبارەی -# instead of -لەباتی -# beside -لەبن -# instead of -لەبرێتی -# behind -لەدەم -# with/together with -لەگەڵ -# by -لەلایەن -# within -لەناو -# between/among -لەنێو -# for the sake of -لەپێناوی -# with respect to -لەرەوی -# by means of/for -لەرێ -# for the sake of -لەرێگا -# on/on top of/according to -لەسەر -# under -لەژێر -# between/among -ناو -# between/among -نێوان -# after -پاش -# before -پێش -# like -وەک diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_cz.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_cz.txt deleted file mode 100644 index 53c6097da..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_cz.txt +++ /dev/null @@ -1,172 +0,0 @@ -a -s -k -o -i -u -v -z -dnes -cz -tímto -budeš -budem -byli -jseš -můj -svým -ta -tomto -tohle -tuto -tyto -jej -zda -proč -máte -tato -kam -tohoto -kdo -kteří -mi -nám -tom -tomuto -mít -nic -proto -kterou -byla -toho -protože -asi -ho -naši -napište -re -což -tím -takže -svých -její -svými -jste -aj -tu -tedy -teto -bylo -kde -ke -pravé -ji -nad -nejsou -či -pod -téma -mezi -přes -ty -pak -vám -ani -když -však -neg -jsem -tento -článku -články -aby -jsme -před -pta -jejich -byl -ještě -až -bez -také -pouze -první -vaše -která -nás -nový -tipy -pokud -může -strana -jeho -své -jiné -zprávy -nové -není -vás -jen -podle -zde -už -být -více -bude -již -než -který -by -které -co -nebo -ten -tak -má -při -od -po -jsou -jak -další -ale -si -se -ve -to -jako -za -zpět -ze -do -pro -je -na -atd -atp -jakmile -přičemž -já -on -ona -ono -oni -ony -my -vy -jí -ji -mě -mne -jemu -tomu -těm -těmu -němu -němuž -jehož -jíž -jelikož -jež -jakož -načež diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_da.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_da.txt deleted file mode 100644 index 42e6145b9..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_da.txt +++ /dev/null @@ -1,110 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/danish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Danish stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This is a ranked list (commonest to rarest) of stopwords derived from - | a large text sample. - - -og | and -i | in -jeg | I -det | that (dem. pronoun)/it (pers. pronoun) -at | that (in front of a sentence)/to (with infinitive) -en | a/an -den | it (pers. pronoun)/that (dem. pronoun) -til | to/at/for/until/against/by/of/into, more -er | present tense of "to be" -som | who, as -på | on/upon/in/on/at/to/after/of/with/for, on -de | they -med | with/by/in, along -han | he -af | of/by/from/off/for/in/with/on, off -for | at/for/to/from/by/of/ago, in front/before, because -ikke | not -der | who/which, there/those -var | past tense of "to be" -mig | me/myself -sig | oneself/himself/herself/itself/themselves -men | but -et | a/an/one, one (number), someone/somebody/one -har | present tense of "to have" -om | round/about/for/in/a, about/around/down, if -vi | we -min | my -havde | past tense of "to have" -ham | him -hun | she -nu | now -over | over/above/across/by/beyond/past/on/about, over/past -da | then, when/as/since -fra | from/off/since, off, since -du | you -ud | out -sin | his/her/its/one's -dem | them -os | us/ourselves -op | up -man | you/one -hans | his -hvor | where -eller | or -hvad | what -skal | must/shall etc. -selv | myself/youself/herself/ourselves etc., even -her | here -alle | all/everyone/everybody etc. -vil | will (verb) -blev | past tense of "to stay/to remain/to get/to become" -kunne | could -ind | in -når | when -være | present tense of "to be" -dog | however/yet/after all -noget | something -ville | would -jo | you know/you see (adv), yes -deres | their/theirs -efter | after/behind/according to/for/by/from, later/afterwards -ned | down -skulle | should -denne | this -end | than -dette | this -mit | my/mine -også | also -under | under/beneath/below/during, below/underneath -have | have -dig | you -anden | other -hende | her -mine | my -alt | everything -meget | much/very, plenty of -sit | his, her, its, one's -sine | his, her, its, one's -vor | our -mod | against -disse | these -hvis | if -din | your/yours -nogle | some -hos | by/at -blive | be/become -mange | many -ad | by/through -bliver | present tense of "to be/to become" -hendes | her/hers -været | be -thi | for (conj) -jer | you -sådan | such, like this/like that diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_de.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_de.txt deleted file mode 100644 index 86525e7ae..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_de.txt +++ /dev/null @@ -1,294 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A German stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | The number of forms in this list is reduced significantly by passing it - | through the German stemmer. - - -aber | but - -alle | all -allem -allen -aller -alles - -als | than, as -also | so -am | an + dem -an | at - -ander | other -andere -anderem -anderen -anderer -anderes -anderm -andern -anderr -anders - -auch | also -auf | on -aus | out of -bei | by -bin | am -bis | until -bist | art -da | there -damit | with it -dann | then - -der | the -den -des -dem -die -das - -daß | that - -derselbe | the same -derselben -denselben -desselben -demselben -dieselbe -dieselben -dasselbe - -dazu | to that - -dein | thy -deine -deinem -deinen -deiner -deines - -denn | because - -derer | of those -dessen | of him - -dich | thee -dir | to thee -du | thou - -dies | this -diese -diesem -diesen -dieser -dieses - - -doch | (several meanings) -dort | (over) there - - -durch | through - -ein | a -eine -einem -einen -einer -eines - -einig | some -einige -einigem -einigen -einiger -einiges - -einmal | once - -er | he -ihn | him -ihm | to him - -es | it -etwas | something - -euer | your -eure -eurem -euren -eurer -eures - -für | for -gegen | towards -gewesen | p.p. of sein -hab | have -habe | have -haben | have -hat | has -hatte | had -hatten | had -hier | here -hin | there -hinter | behind - -ich | I -mich | me -mir | to me - - -ihr | you, to her -ihre -ihrem -ihren -ihrer -ihres -euch | to you - -im | in + dem -in | in -indem | while -ins | in + das -ist | is - -jede | each, every -jedem -jeden -jeder -jedes - -jene | that -jenem -jenen -jener -jenes - -jetzt | now -kann | can - -kein | no -keine -keinem -keinen -keiner -keines - -können | can -könnte | could -machen | do -man | one - -manche | some, many a -manchem -manchen -mancher -manches - -mein | my -meine -meinem -meinen -meiner -meines - -mit | with -muss | must -musste | had to -nach | to(wards) -nicht | not -nichts | nothing -noch | still, yet -nun | now -nur | only -ob | whether -oder | or -ohne | without -sehr | very - -sein | his -seine -seinem -seinen -seiner -seines - -selbst | self -sich | herself - -sie | they, she -ihnen | to them - -sind | are -so | so - -solche | such -solchem -solchen -solcher -solches - -soll | shall -sollte | should -sondern | but -sonst | else -über | over -um | about, around -und | and - -uns | us -unse -unsem -unsen -unser -unses - -unter | under -viel | much -vom | von + dem -von | from -vor | before -während | while -war | was -waren | were -warst | wast -was | what -weg | away, off -weil | because -weiter | further - -welche | which -welchem -welchen -welcher -welches - -wenn | when -werde | will -werden | will -wie | how -wieder | again -will | want -wir | we -wird | will -wirst | willst -wo | where -wollen | want -wollte | wanted -würde | would -würden | would -zu | to -zum | zu + dem -zur | zu + der -zwar | indeed -zwischen | between - diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_el.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_el.txt deleted file mode 100644 index 232681f5b..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_el.txt +++ /dev/null @@ -1,78 +0,0 @@ -# Lucene Greek Stopwords list -# Note: by default this file is used after GreekLowerCaseFilter, -# so when modifying this file use 'σ' instead of 'ς' -ο -η -το -οι -τα -του -τησ -των -τον -την -και -κι -κ -ειμαι -εισαι -ειναι -ειμαστε -ειστε -στο -στον -στη -στην -μα -αλλα -απο -για -προσ -με -σε -ωσ -παρα -αντι -κατα -μετα -θα -να -δε -δεν -μη -μην -επι -ενω -εαν -αν -τοτε -που -πωσ -ποιοσ -ποια -ποιο -ποιοι -ποιεσ -ποιων -ποιουσ -αυτοσ -αυτη -αυτο -αυτοι -αυτων -αυτουσ -αυτεσ -αυτα -εκεινοσ -εκεινη -εκεινο -εκεινοι -εκεινεσ -εκεινα -εκεινων -εκεινουσ -οπωσ -ομωσ -ισωσ -οσο -οτι diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_en.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_en.txt deleted file mode 100644 index 8bb2b7de4..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_en.txt +++ /dev/null @@ -1,332 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -### Top 200 words based on frequency in common use + stemmed versions -### > 1000 per million words -the -of -and -a -in -to -it -is -was -wa -I -for -that -you -he -be -with -on -by -at -have -are -ar -not -this -thi -but -had -they -thei -his -hi -from - -### > 3000 PMW -she -which -or -we -an -were -as - -### > 2000 PMW -do -been -their -has -ha -would -there -what -will -all -if -can -her -#said -who - -### Top 50 -### > 1000 PMW -#one -#on -so -up -them -some -when -could -him -into -its -it -then -#two -out -#time -my -about -#did -#your -#now -me -no -other -only -onli -#just -more -these -also -#people -#peopl -#know -any -ani -#first -#see -very -veri -new -#may -#mai -#well -should -#like -than -how - -### Top 90 -### > 900 PMW -#get -#way -#wai -our -#made -#got -after -#think -between -#many -#mani -#years -#year - -### Top 100 -### > 800 PMW -er -those -#go -being -be -because -becaus -down -#yeah - -### > 700 PMW -#three -#good -#back -#make -such -#through -#year -#over -#must -#still -#even -#take -#too - -### Top 120 above - -### > 600 PMW -#here -#come -#own -#last -#does -#doe -#oh -#say -#sai -#work -#where -#erm -#us -#government -#govern -#same -#man -#might -#day -#dai -#yes -#ye -#however -#howev - -### > 500 PMW -#put -#world -#another -#anoth -#want -#life -#most -#against -#again -#never -#under -#old -#much -#something -#someth -#Mr -#why -#each -#while -#house -#hous - -### > 400 PMW -#part -#number -#out of -#found -#off -#different -#differ -#went -#really -#realli -#thought -#came -#used -#us -#children -#always -#alwai -#four -#without -#give -#few -#within -#system -#local -#place -#great -#during -#dure -#although -#small -#before -#befor -#look -#case -#next -#end -#things -#thing -#social -#find -#group -#quite -#quit -#mean -#five -#party -#parti -#company -#compani -#every -#everi - -### > 300 PMW -#women -#says -#sai -#important -#import -#took - -### Top 200 - - - - - - -###################### Lucene defaults .... - - -# Standard english stop words taken from Lucene's StopAnalyzer -# - Added stemmed varients - are -> ar, they -> thei, this -> thi, was -> wa -#a -#an -#and -#are -#ar -#as -#at -#be -#but -#by -#for -#if -#in -#into -#is -#it -#no -#not -#of -#on -#or -#such -#that -#the -#their -#then -#there -#these -#they -#thei -#this -#thi -#to -#was -#wa -#will -#with diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_es.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_es.txt deleted file mode 100644 index 487d78c8d..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_es.txt +++ /dev/null @@ -1,356 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Spanish stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - - | The following is a ranked list (commonest to rarest) of stopwords - | deriving from a large sample of text. - - | Extra words have been added at the end. - -de | from, of -la | the, her -que | who, that -el | the -en | in -y | and -a | to -los | the, them -del | de + el -se | himself, from him etc -las | the, them -por | for, by, etc -un | a -para | for -con | with -no | no -una | a -su | his, her -al | a + el - | es from SER -lo | him -como | how -más | more -pero | pero -sus | su plural -le | to him, her -ya | already -o | or - | fue from SER -este | this - | ha from HABER -sí | himself etc -porque | because -esta | this - | son from SER -entre | between - | está from ESTAR -cuando | when -muy | very -sin | without -sobre | on - | ser from SER - | tiene from TENER -también | also -me | me -hasta | until -hay | there is/are -donde | where - | han from HABER -quien | whom, that - | están from ESTAR - | estado from ESTAR -desde | from -todo | all -nos | us -durante | during - | estados from ESTAR -todos | all -uno | a -les | to them -ni | nor -contra | against -otros | other - | fueron from SER -ese | that -eso | that - | había from HABER -ante | before -ellos | they -e | and (variant of y) -esto | this -mí | me -antes | before -algunos | some -qué | what? -unos | a -yo | I -otro | other -otras | other -otra | other -él | he -tanto | so much, many -esa | that -estos | these -mucho | much, many -quienes | who -nada | nothing -muchos | many -cual | who - | sea from SER -poco | few -ella | she -estar | to be - | haber from HABER -estas | these - | estaba from ESTAR - | estamos from ESTAR -algunas | some -algo | something -nosotros | we - - | other forms - -mi | me -mis | mi plural -tú | thou -te | thee -ti | thee -tu | thy -tus | tu plural -ellas | they -nosotras | we -vosotros | you -vosotras | you -os | you -mío | mine -mía | -míos | -mías | -tuyo | thine -tuya | -tuyos | -tuyas | -suyo | his, hers, theirs -suya | -suyos | -suyas | -nuestro | ours -nuestra | -nuestros | -nuestras | -vuestro | yours -vuestra | -vuestros | -vuestras | -esos | those -esas | those - - | forms of estar, to be (not including the infinitive): -estoy -estás -está -estamos -estáis -están -esté -estés -estemos -estéis -estén -estaré -estarás -estará -estaremos -estaréis -estarán -estaría -estarías -estaríamos -estaríais -estarían -estaba -estabas -estábamos -estabais -estaban -estuve -estuviste -estuvo -estuvimos -estuvisteis -estuvieron -estuviera -estuvieras -estuviéramos -estuvierais -estuvieran -estuviese -estuvieses -estuviésemos -estuvieseis -estuviesen -estando -estado -estada -estados -estadas -estad - - | forms of haber, to have (not including the infinitive): -he -has -ha -hemos -habéis -han -haya -hayas -hayamos -hayáis -hayan -habré -habrás -habrá -habremos -habréis -habrán -habría -habrías -habríamos -habríais -habrían -había -habías -habíamos -habíais -habían -hube -hubiste -hubo -hubimos -hubisteis -hubieron -hubiera -hubieras -hubiéramos -hubierais -hubieran -hubiese -hubieses -hubiésemos -hubieseis -hubiesen -habiendo -habido -habida -habidos -habidas - - | forms of ser, to be (not including the infinitive): -soy -eres -es -somos -sois -son -sea -seas -seamos -seáis -sean -seré -serás -será -seremos -seréis -serán -sería -serías -seríamos -seríais -serían -era -eras -éramos -erais -eran -fui -fuiste -fue -fuimos -fuisteis -fueron -fuera -fueras -fuéramos -fuerais -fueran -fuese -fueses -fuésemos -fueseis -fuesen -siendo -sido - | sed also means 'thirst' - - | forms of tener, to have (not including the infinitive): -tengo -tienes -tiene -tenemos -tenéis -tienen -tenga -tengas -tengamos -tengáis -tengan -tendré -tendrás -tendrá -tendremos -tendréis -tendrán -tendría -tendrías -tendríamos -tendríais -tendrían -tenía -tenías -teníamos -teníais -tenían -tuve -tuviste -tuvo -tuvimos -tuvisteis -tuvieron -tuviera -tuvieras -tuviéramos -tuvierais -tuvieran -tuviese -tuvieses -tuviésemos -tuvieseis -tuviesen -teniendo -tenido -tenida -tenidos -tenidas -tened - diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_eu.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_eu.txt deleted file mode 100644 index 25f1db934..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_eu.txt +++ /dev/null @@ -1,99 +0,0 @@ -# example set of basque stopwords -al -anitz -arabera -asko -baina -bat -batean -batek -bati -batzuei -batzuek -batzuetan -batzuk -bera -beraiek -berau -berauek -bere -berori -beroriek -beste -bezala -da -dago -dira -ditu -du -dute -edo -egin -ere -eta -eurak -ez -gainera -gu -gutxi -guzti -haiei -haiek -haietan -hainbeste -hala -han -handik -hango -hara -hari -hark -hartan -hau -hauei -hauek -hauetan -hemen -hemendik -hemengo -hi -hona -honek -honela -honetan -honi -hor -hori -horiei -horiek -horietan -horko -horra -horrek -horrela -horretan -horri -hortik -hura -izan -ni -noiz -nola -non -nondik -nongo -nor -nora -ze -zein -zen -zenbait -zenbat -zer -zergatik -ziren -zituen -zu -zuek -zuen -zuten diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fa.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fa.txt deleted file mode 100644 index 723641c6d..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fa.txt +++ /dev/null @@ -1,313 +0,0 @@ -# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -# Note: by default this file is used after normalization, so when adding entries -# to this file, use the arabic 'ي' instead of 'ی' -انان -نداشته -سراسر -خياه -ايشان -وي -تاكنون -بيشتري -دوم -پس -ناشي -وگو -يا -داشتند -سپس -هنگام -هرگز -پنج -نشان -امسال -ديگر -گروهي -شدند -چطور -ده -و -دو -نخستين -ولي -چرا -چه -وسط -ه -كدام -قابل -يك -رفت -هفت -همچنين -در -هزار -بله -بلي -شايد -اما -شناسي -گرفته -دهد -داشته -دانست -داشتن -خواهيم -ميليارد -وقتيكه -امد -خواهد -جز -اورده -شده -بلكه -خدمات -شدن -برخي -نبود -بسياري -جلوگيري -حق -كردند -نوعي -بعري -نكرده -نظير -نبايد -بوده -بودن -داد -اورد -هست -جايي -شود -دنبال -داده -بايد -سابق -هيچ -همان -انجا -كمتر -كجاست -گردد -كسي -تر -مردم -تان -دادن -بودند -سري -جدا -ندارند -مگر -يكديگر -دارد -دهند -بنابراين -هنگامي -سمت -جا -انچه -خود -دادند -زياد -دارند -اثر -بدون -بهترين -بيشتر -البته -به -براساس -بيرون -كرد -بعضي -گرفت -توي -اي -ميليون -او -جريان -تول -بر -مانند -برابر -باشيم -مدتي -گويند -اكنون -تا -تنها -جديد -چند -بي -نشده -كردن -كردم -گويد -كرده -كنيم -نمي -نزد -روي -قصد -فقط -بالاي -ديگران -اين -ديروز -توسط -سوم -ايم -دانند -سوي -استفاده -شما -كنار -داريم -ساخته -طور -امده -رفته -نخست -بيست -نزديك -طي -كنيد -از -انها -تمامي -داشت -يكي -طريق -اش -چيست -روب -نمايد -گفت -چندين -چيزي -تواند -ام -ايا -با -ان -ايد -ترين -اينكه -ديگري -راه -هايي -بروز -همچنان -پاعين -كس -حدود -مختلف -مقابل -چيز -گيرد -ندارد -ضد -همچون -سازي -شان -مورد -باره -مرسي -خويش -برخوردار -چون -خارج -شش -هنوز -تحت -ضمن -هستيم -گفته -فكر -بسيار -پيش -براي -روزهاي -انكه -نخواهد -بالا -كل -وقتي -كي -چنين -كه -گيري -نيست -است -كجا -كند -نيز -يابد -بندي -حتي -توانند -عقب -خواست -كنند -بين -تمام -همه -ما -باشند -مثل -شد -اري -باشد -اره -طبق -بعد -اگر -صورت -غير -جاي -بيش -ريزي -اند -زيرا -چگونه -بار -لطفا -مي -درباره -من -ديده -همين -گذاري -برداري -علت -گذاشته -هم -فوق -نه -ها -شوند -اباد -همواره -هر -اول -خواهند -چهار -نام -امروز -مان -هاي -قبل -كنم -سعي -تازه -را -هستند -زير -جلوي -عنوان -بود diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fi.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fi.txt deleted file mode 100644 index 4372c9a05..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fi.txt +++ /dev/null @@ -1,97 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - -| forms of BE - -olla -olen -olet -on -olemme -olette -ovat -ole | negative form - -oli -olisi -olisit -olisin -olisimme -olisitte -olisivat -olit -olin -olimme -olitte -olivat -ollut -olleet - -en | negation -et -ei -emme -ette -eivät - -|Nom Gen Acc Part Iness Elat Illat Adess Ablat Allat Ess Trans -minä minun minut minua minussa minusta minuun minulla minulta minulle | I -sinä sinun sinut sinua sinussa sinusta sinuun sinulla sinulta sinulle | you -hän hänen hänet häntä hänessä hänestä häneen hänellä häneltä hänelle | he she -me meidän meidät meitä meissä meistä meihin meillä meiltä meille | we -te teidän teidät teitä teissä teistä teihin teillä teiltä teille | you -he heidän heidät heitä heissä heistä heihin heillä heiltä heille | they - -tämä tämän tätä tässä tästä tähän tallä tältä tälle tänä täksi | this -tuo tuon tuotä tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that -se sen sitä siinä siitä siihen sillä siltä sille sinä siksi | it -nämä näiden näitä näissä näistä näihin näillä näiltä näille näinä näiksi | these -nuo noiden noita noissa noista noihin noilla noilta noille noina noiksi | those -ne niiden niitä niissä niistä niihin niillä niiltä niille niinä niiksi | they - -kuka kenen kenet ketä kenessä kenestä keneen kenellä keneltä kenelle kenenä keneksi| who -ketkä keiden ketkä keitä keissä keistä keihin keillä keiltä keille keinä keiksi | (pl) -mikä minkä minkä mitä missä mistä mihin millä miltä mille minä miksi | which what -mitkä | (pl) - -joka jonka jota jossa josta johon jolla jolta jolle jona joksi | who which -jotka joiden joita joissa joista joihin joilla joilta joille joina joiksi | (pl) - -| conjunctions - -että | that -ja | and -jos | if -koska | because -kuin | than -mutta | but -niin | so -sekä | and -sillä | for -tai | or -vaan | but -vai | or -vaikka | although - - -| prepositions - -kanssa | with -mukaan | according to -noin | about -poikki | across -yli | over, across - -| other - -kun | when -niin | so -nyt | now -itse | self - diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fr.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fr.txt deleted file mode 100644 index 749abae68..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_fr.txt +++ /dev/null @@ -1,186 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A French stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - -au | a + le -aux | a + les -avec | with -ce | this -ces | these -dans | with -de | of -des | de + les -du | de + le -elle | she -en | `of them' etc -et | and -eux | them -il | he -je | I -la | the -le | the -leur | their -lui | him -ma | my (fem) -mais | but -me | me -même | same; as in moi-même (myself) etc -mes | me (pl) -moi | me -mon | my (masc) -ne | not -nos | our (pl) -notre | our -nous | we -on | one -ou | where -par | by -pas | not -pour | for -qu | que before vowel -que | that -qui | who -sa | his, her (fem) -se | oneself -ses | his (pl) -son | his, her (masc) -sur | on -ta | thy (fem) -te | thee -tes | thy (pl) -toi | thee -ton | thy (masc) -tu | thou -un | a -une | a -vos | your (pl) -votre | your -vous | you - - | single letter forms - -c | c' -d | d' -j | j' -l | l' -à | to, at -m | m' -n | n' -s | s' -t | t' -y | there - - | forms of être (not including the infinitive): -été -étée -étées -étés -étant -suis -es -est -sommes -êtes -sont -serai -seras -sera -serons -serez -seront -serais -serait -serions -seriez -seraient -étais -était -étions -étiez -étaient -fus -fut -fûmes -fûtes -furent -sois -soit -soyons -soyez -soient -fusse -fusses -fût -fussions -fussiez -fussent - - | forms of avoir (not including the infinitive): -ayant -eu -eue -eues -eus -ai -as -avons -avez -ont -aurai -auras -aura -aurons -aurez -auront -aurais -aurait -aurions -auriez -auraient -avais -avait -avions -aviez -avaient -eut -eûmes -eûtes -eurent -aie -aies -ait -ayons -ayez -aient -eusse -eusses -eût -eussions -eussiez -eussent - - | Later additions (from Jean-Christophe Deschamps) -ceci | this -cela | that -celà | that -cet | this -cette | this -ici | here -ils | they -les | the (pl) -leurs | their (pl) -quel | which -quels | which -quelle | which -quelles | which -sans | without -soi | oneself - diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ga.txt deleted file mode 100644 index 9ff88d747..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ga.txt +++ /dev/null @@ -1,110 +0,0 @@ - -a -ach -ag -agus -an -aon -ar -arna -as -b' -ba -beirt -bhúr -caoga -ceathair -ceathrar -chomh -chtó -chuig -chun -cois -céad -cúig -cúigear -d' -daichead -dar -de -deich -deichniúr -den -dhá -do -don -dtí -dá -dár -dó -faoi -faoin -faoina -faoinár -fara -fiche -gach -gan -go -gur -haon -hocht -i -iad -idir -in -ina -ins -inár -is -le -leis -lena -lenár -m' -mar -mo -mé -na -nach -naoi -naonúr -ná -ní -níor -nó -nócha -ocht -ochtar -os -roimh -sa -seacht -seachtar -seachtó -seasca -seisear -siad -sibh -sinn -sna -sé -sí -tar -thar -thú -triúr -trí -trína -trínár -tríocha -tú -um -ár -é -éis -í -ó -ón -óna -ónár diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_gl.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_gl.txt deleted file mode 100644 index d8760b12c..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_gl.txt +++ /dev/null @@ -1,161 +0,0 @@ -# galican stopwords -a -aínda -alí -aquel -aquela -aquelas -aqueles -aquilo -aquí -ao -aos -as -así -á -ben -cando -che -co -coa -comigo -con -connosco -contigo -convosco -coas -cos -cun -cuns -cunha -cunhas -da -dalgunha -dalgunhas -dalgún -dalgúns -das -de -del -dela -delas -deles -desde -deste -do -dos -dun -duns -dunha -dunhas -e -el -ela -elas -eles -en -era -eran -esa -esas -ese -eses -esta -estar -estaba -está -están -este -estes -estiven -estou -eu -é -facer -foi -foron -fun -había -hai -iso -isto -la -las -lle -lles -lo -los -mais -me -meu -meus -min -miña -miñas -moi -na -nas -neste -nin -no -non -nos -nosa -nosas -noso -nosos -nós -nun -nunha -nuns -nunhas -o -os -ou -ó -ós -para -pero -pode -pois -pola -polas -polo -polos -por -que -se -senón -ser -seu -seus -sexa -sido -sobre -súa -súas -tamén -tan -te -ten -teñen -teño -ter -teu -teus -ti -tido -tiña -tiven -túa -túas -un -unha -unhas -uns -vos -vosa -vosas -voso -vosos -vós diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hi.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hi.txt deleted file mode 100644 index 86286bb08..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hi.txt +++ /dev/null @@ -1,235 +0,0 @@ -# Also see http://www.opensource.org/licenses/bsd-license.html -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# This file was created by Jacques Savoy and is distributed under the BSD license. -# Note: by default this file also contains forms normalized by HindiNormalizer -# for spelling variation (see section below), such that it can be used whether or -# not you enable that feature. When adding additional entries to this list, -# please add the normalized form as well. -अंदर -अत -अपना -अपनी -अपने -अभी -आदि -आप -इत्यादि -इन -इनका -इन्हीं -इन्हें -इन्हों -इस -इसका -इसकी -इसके -इसमें -इसी -इसे -उन -उनका -उनकी -उनके -उनको -उन्हीं -उन्हें -उन्हों -उस -उसके -उसी -उसे -एक -एवं -एस -ऐसे -और -कई -कर -करता -करते -करना -करने -करें -कहते -कहा -का -काफ़ी -कि -कितना -किन्हें -किन्हों -किया -किर -किस -किसी -किसे -की -कुछ -कुल -के -को -कोई -कौन -कौनसा -गया -घर -जब -जहाँ -जा -जितना -जिन -जिन्हें -जिन्हों -जिस -जिसे -जीधर -जैसा -जैसे -जो -तक -तब -तरह -तिन -तिन्हें -तिन्हों -तिस -तिसे -तो -था -थी -थे -दबारा -दिया -दुसरा -दूसरे -दो -द्वारा -न -नहीं -ना -निहायत -नीचे -ने -पर -पर -पहले -पूरा -पे -फिर -बनी -बही -बहुत -बाद -बाला -बिलकुल -भी -भीतर -मगर -मानो -मे -में -यदि -यह -यहाँ -यही -या -यिह -ये -रखें -रहा -रहे -ऱ्वासा -लिए -लिये -लेकिन -व -वर्ग -वह -वह -वहाँ -वहीं -वाले -वुह -वे -वग़ैरह -संग -सकता -सकते -सबसे -सभी -साथ -साबुत -साभ -सारा -से -सो -ही -हुआ -हुई -हुए -है -हैं -हो -होता -होती -होते -होना -होने -# additional normalized forms of the above -अपनि -जेसे -होति -सभि -तिंहों -इंहों -दवारा -इसि -किंहें -थि -उंहों -ओर -जिंहें -वहिं -अभि -बनि -हि -उंहिं -उंहें -हें -वगेरह -एसे -रवासा -कोन -निचे -काफि -उसि -पुरा -भितर -हे -बहि -वहां -कोइ -यहां -जिंहों -तिंहें -किसि -कइ -यहि -इंहिं -जिधर -इंहें -अदि -इतयादि -हुइ -कोनसा -इसकि -दुसरे -जहां -अप -किंहों -उनकि -भि -वरग -हुअ -जेसा -नहिं diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hu.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hu.txt deleted file mode 100644 index 37526da8a..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hu.txt +++ /dev/null @@ -1,211 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/hungarian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - -| Hungarian stop word list -| prepared by Anna Tordai - -a -ahogy -ahol -aki -akik -akkor -alatt -által -általában -amely -amelyek -amelyekben -amelyeket -amelyet -amelynek -ami -amit -amolyan -amíg -amikor -át -abban -ahhoz -annak -arra -arról -az -azok -azon -azt -azzal -azért -aztán -azután -azonban -bár -be -belül -benne -cikk -cikkek -cikkeket -csak -de -e -eddig -egész -egy -egyes -egyetlen -egyéb -egyik -egyre -ekkor -el -elég -ellen -elő -először -előtt -első -én -éppen -ebben -ehhez -emilyen -ennek -erre -ez -ezt -ezek -ezen -ezzel -ezért -és -fel -felé -hanem -hiszen -hogy -hogyan -igen -így -illetve -ill. -ill -ilyen -ilyenkor -ison -ismét -itt -jó -jól -jobban -kell -kellett -keresztül -keressünk -ki -kívül -között -közül -legalább -lehet -lehetett -legyen -lenne -lenni -lesz -lett -maga -magát -majd -majd -már -más -másik -meg -még -mellett -mert -mely -melyek -mi -mit -míg -miért -milyen -mikor -minden -mindent -mindenki -mindig -mint -mintha -mivel -most -nagy -nagyobb -nagyon -ne -néha -nekem -neki -nem -néhány -nélkül -nincs -olyan -ott -össze -ő -ők -őket -pedig -persze -rá -s -saját -sem -semmi -sok -sokat -sokkal -számára -szemben -szerint -szinte -talán -tehát -teljes -tovább -továbbá -több -úgy -ugyanis -új -újabb -újra -után -utána -utolsó -vagy -vagyis -valaki -valami -valamint -való -vagyok -van -vannak -volt -voltam -voltak -voltunk -vissza -vele -viszont -volna diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hy.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hy.txt deleted file mode 100644 index 60c1c50fb..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_hy.txt +++ /dev/null @@ -1,46 +0,0 @@ -# example set of Armenian stopwords. -այդ -այլ -այն -այս -դու -դուք -եմ -են -ենք -ես -եք -է -էի -էին -էինք -էիր -էիք -էր -ըստ -թ -ի -ին -իսկ -իր -կամ -համար -հետ -հետո -մենք -մեջ -մի -ն -նա -նաև -նրա -նրանք -որ -որը -որոնք -որպես -ու -ում -պիտի -վրա -և diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_id.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_id.txt deleted file mode 100644 index 4617f83a5..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_id.txt +++ /dev/null @@ -1,359 +0,0 @@ -# from appendix D of: A Study of Stemming Effects on Information -# Retrieval in Bahasa Indonesia -ada -adanya -adalah -adapun -agak -agaknya -agar -akan -akankah -akhirnya -aku -akulah -amat -amatlah -anda -andalah -antar -diantaranya -antara -antaranya -diantara -apa -apaan -mengapa -apabila -apakah -apalagi -apatah -atau -ataukah -ataupun -bagai -bagaikan -sebagai -sebagainya -bagaimana -bagaimanapun -sebagaimana -bagaimanakah -bagi -bahkan -bahwa -bahwasanya -sebaliknya -banyak -sebanyak -beberapa -seberapa -begini -beginian -beginikah -beginilah -sebegini -begitu -begitukah -begitulah -begitupun -sebegitu -belum -belumlah -sebelum -sebelumnya -sebenarnya -berapa -berapakah -berapalah -berapapun -betulkah -sebetulnya -biasa -biasanya -bila -bilakah -bisa -bisakah -sebisanya -boleh -bolehkah -bolehlah -buat -bukan -bukankah -bukanlah -bukannya -cuma -percuma -dahulu -dalam -dan -dapat -dari -daripada -dekat -demi -demikian -demikianlah -sedemikian -dengan -depan -di -dia -dialah -dini -diri -dirinya -terdiri -dong -dulu -enggak -enggaknya -entah -entahlah -terhadap -terhadapnya -hal -hampir -hanya -hanyalah -harus -haruslah -harusnya -seharusnya -hendak -hendaklah -hendaknya -hingga -sehingga -ia -ialah -ibarat -ingin -inginkah -inginkan -ini -inikah -inilah -itu -itukah -itulah -jangan -jangankan -janganlah -jika -jikalau -juga -justru -kala -kalau -kalaulah -kalaupun -kalian -kami -kamilah -kamu -kamulah -kan -kapan -kapankah -kapanpun -dikarenakan -karena -karenanya -ke -kecil -kemudian -kenapa -kepada -kepadanya -ketika -seketika -khususnya -kini -kinilah -kiranya -sekiranya -kita -kitalah -kok -lagi -lagian -selagi -lah -lain -lainnya -melainkan -selaku -lalu -melalui -terlalu -lama -lamanya -selama -selama -selamanya -lebih -terlebih -bermacam -macam -semacam -maka -makanya -makin -malah -malahan -mampu -mampukah -mana -manakala -manalagi -masih -masihkah -semasih -masing -mau -maupun -semaunya -memang -mereka -merekalah -meski -meskipun -semula -mungkin -mungkinkah -nah -namun -nanti -nantinya -nyaris -oleh -olehnya -seorang -seseorang -pada -padanya -padahal -paling -sepanjang -pantas -sepantasnya -sepantasnyalah -para -pasti -pastilah -per -pernah -pula -pun -merupakan -rupanya -serupa -saat -saatnya -sesaat -saja -sajalah -saling -bersama -sama -sesama -sambil -sampai -sana -sangat -sangatlah -saya -sayalah -se -sebab -sebabnya -sebuah -tersebut -tersebutlah -sedang -sedangkan -sedikit -sedikitnya -segala -segalanya -segera -sesegera -sejak -sejenak -sekali -sekalian -sekalipun -sesekali -sekaligus -sekarang -sekarang -sekitar -sekitarnya -sela -selain -selalu -seluruh -seluruhnya -semakin -sementara -sempat -semua -semuanya -sendiri -sendirinya -seolah -seperti -sepertinya -sering -seringnya -serta -siapa -siapakah -siapapun -disini -disinilah -sini -sinilah -sesuatu -sesuatunya -suatu -sesudah -sesudahnya -sudah -sudahkah -sudahlah -supaya -tadi -tadinya -tak -tanpa -setelah -telah -tentang -tentu -tentulah -tentunya -tertentu -seterusnya -tapi -tetapi -setiap -tiap -setidaknya -tidak -tidakkah -tidaklah -toh -waduh -wah -wahai -sewaktu -walau -walaupun -wong -yaitu -yakni -yang diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_it.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_it.txt deleted file mode 100644 index 1219cc773..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_it.txt +++ /dev/null @@ -1,303 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | An Italian stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - -ad | a (to) before vowel -al | a + il -allo | a + lo -ai | a + i -agli | a + gli -all | a + l' -agl | a + gl' -alla | a + la -alle | a + le -con | with -col | con + il -coi | con + i (forms collo, cogli etc are now very rare) -da | from -dal | da + il -dallo | da + lo -dai | da + i -dagli | da + gli -dall | da + l' -dagl | da + gll' -dalla | da + la -dalle | da + le -di | of -del | di + il -dello | di + lo -dei | di + i -degli | di + gli -dell | di + l' -degl | di + gl' -della | di + la -delle | di + le -in | in -nel | in + el -nello | in + lo -nei | in + i -negli | in + gli -nell | in + l' -negl | in + gl' -nella | in + la -nelle | in + le -su | on -sul | su + il -sullo | su + lo -sui | su + i -sugli | su + gli -sull | su + l' -sugl | su + gl' -sulla | su + la -sulle | su + le -per | through, by -tra | among -contro | against -io | I -tu | thou -lui | he -lei | she -noi | we -voi | you -loro | they -mio | my -mia | -miei | -mie | -tuo | -tua | -tuoi | thy -tue | -suo | -sua | -suoi | his, her -sue | -nostro | our -nostra | -nostri | -nostre | -vostro | your -vostra | -vostri | -vostre | -mi | me -ti | thee -ci | us, there -vi | you, there -lo | him, the -la | her, the -li | them -le | them, the -gli | to him, the -ne | from there etc -il | the -un | a -uno | a -una | a -ma | but -ed | and -se | if -perché | why, because -anche | also -come | how -dov | where (as dov') -dove | where -che | who, that -chi | who -cui | whom -non | not -più | more -quale | who, that -quanto | how much -quanti | -quanta | -quante | -quello | that -quelli | -quella | -quelle | -questo | this -questi | -questa | -queste | -si | yes -tutto | all -tutti | all - - | single letter forms: - -a | at -c | as c' for ce or ci -e | and -i | the -l | as l' -o | or - - | forms of avere, to have (not including the infinitive): - -ho -hai -ha -abbiamo -avete -hanno -abbia -abbiate -abbiano -avrò -avrai -avrà -avremo -avrete -avranno -avrei -avresti -avrebbe -avremmo -avreste -avrebbero -avevo -avevi -aveva -avevamo -avevate -avevano -ebbi -avesti -ebbe -avemmo -aveste -ebbero -avessi -avesse -avessimo -avessero -avendo -avuto -avuta -avuti -avute - - | forms of essere, to be (not including the infinitive): -sono -sei -è -siamo -siete -sia -siate -siano -sarò -sarai -sarà -saremo -sarete -saranno -sarei -saresti -sarebbe -saremmo -sareste -sarebbero -ero -eri -era -eravamo -eravate -erano -fui -fosti -fu -fummo -foste -furono -fossi -fosse -fossimo -fossero -essendo - - | forms of fare, to do (not including the infinitive, fa, fat-): -faccio -fai -facciamo -fanno -faccia -facciate -facciano -farò -farai -farà -faremo -farete -faranno -farei -faresti -farebbe -faremmo -fareste -farebbero -facevo -facevi -faceva -facevamo -facevate -facevano -feci -facesti -fece -facemmo -faceste -fecero -facessi -facesse -facessimo -facessero -facendo - - | forms of stare, to be (not including the infinitive): -sto -stai -sta -stiamo -stanno -stia -stiate -stiano -starò -starai -starà -staremo -starete -staranno -starei -staresti -starebbe -staremmo -stareste -starebbero -stavo -stavi -stava -stavamo -stavate -stavano -stetti -stesti -stette -stemmo -steste -stettero -stessi -stesse -stessimo -stessero -stando diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ja.txt deleted file mode 100644 index d4321be6b..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ja.txt +++ /dev/null @@ -1,127 +0,0 @@ -# -# This file defines a stopword set for Japanese. -# -# This set is made up of hand-picked frequent terms from segmented Japanese Wikipedia. -# Punctuation characters and frequent kanji have mostly been left out. See LUCENE-3745 -# for frequency lists, etc. that can be useful for making your own set (if desired) -# -# Note that there is an overlap between these stopwords and the terms stopped when used -# in combination with the JapanesePartOfSpeechStopFilter. When editing this file, note -# that comments are not allowed on the same line as stopwords. -# -# Also note that stopping is done in a case-insensitive manner. Change your StopFilter -# configuration if you need case-sensitive stopping. Lastly, note that stopping is done -# using the same character width as the entries in this file. Since this StopFilter is -# normally done after a CJKWidthFilter in your chain, you would usually want your romaji -# entries to be in half-width and your kana entries to be in full-width. -# -の -に -は -を -た -が -で -て -と -し -れ -さ -ある -いる -も -する -から -な -こと -として -い -や -れる -など -なっ -ない -この -ため -その -あっ -よう -また -もの -という -あり -まで -られ -なる -へ -か -だ -これ -によって -により -おり -より -による -ず -なり -られる -において -ば -なかっ -なく -しかし -について -せ -だっ -その後 -できる -それ -う -ので -なお -のみ -でき -き -つ -における -および -いう -さらに -でも -ら -たり -その他 -に関する -たち -ます -ん -なら -に対して -特に -せる -及び -これら -とき -では -にて -ほか -ながら -うち -そして -とともに -ただし -かつて -それぞれ -または -お -ほど -ものの -に対する -ほとんど -と共に -といった -です -とも -ところ -ここ -##### End of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_lv.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_lv.txt deleted file mode 100644 index e21a23c06..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_lv.txt +++ /dev/null @@ -1,172 +0,0 @@ -# Set of Latvian stopwords from A Stemming Algorithm for Latvian, Karlis Kreslins -# the original list of over 800 forms was refined: -# pronouns, adverbs, interjections were removed -# -# prepositions -aiz -ap -ar -apakš -ārpus -augšpus -bez -caur -dēļ -gar -iekš -iz -kopš -labad -lejpus -līdz -no -otrpus -pa -par -pār -pēc -pie -pirms -pret -priekš -starp -šaipus -uz -viņpus -virs -virspus -zem -apakšpus -# Conjunctions -un -bet -jo -ja -ka -lai -tomēr -tikko -turpretī -arī -kaut -gan -tādēļ -tā -ne -tikvien -vien -kā -ir -te -vai -kamēr -# Particles -ar -diezin -droši -diemžēl -nebūt -ik -it -taču -nu -pat -tiklab -iekšpus -nedz -tik -nevis -turpretim -jeb -iekam -iekām -iekāms -kolīdz -līdzko -tiklīdz -jebšu -tālab -tāpēc -nekā -itin -jā -jau -jel -nē -nezin -tad -tikai -vis -tak -iekams -vien -# modal verbs -būt -biju -biji -bija -bijām -bijāt -esmu -esi -esam -esat -būšu -būsi -būs -būsim -būsiet -tikt -tiku -tiki -tika -tikām -tikāt -tieku -tiec -tiek -tiekam -tiekat -tikšu -tiks -tiksim -tiksiet -tapt -tapi -tapāt -topat -tapšu -tapsi -taps -tapsim -tapsiet -kļūt -kļuvu -kļuvi -kļuva -kļuvām -kļuvāt -kļūstu -kļūsti -kļūst -kļūstam -kļūstat -kļūšu -kļūsi -kļūs -kļūsim -kļūsiet -# verbs -varēt -varēju -varējām -varēšu -varēsim -var -varēji -varējāt -varēsi -varēsiet -varat -varēja -varēs diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_nl.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_nl.txt deleted file mode 100644 index 47a2aeacf..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_nl.txt +++ /dev/null @@ -1,119 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Dutch stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This is a ranked list (commonest to rarest) of stopwords derived from - | a large sample of Dutch text. - - | Dutch stop words frequently exhibit homonym clashes. These are indicated - | clearly below. - -de | the -en | and -van | of, from -ik | I, the ego -te | (1) chez, at etc, (2) to, (3) too -dat | that, which -die | that, those, who, which -in | in, inside -een | a, an, one -hij | he -het | the, it -niet | not, nothing, naught -zijn | (1) to be, being, (2) his, one's, its -is | is -was | (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river -op | on, upon, at, in, up, used up -aan | on, upon, to (as dative) -met | with, by -als | like, such as, when -voor | (1) before, in front of, (2) furrow -had | had, past tense all persons sing. of 'hebben' (have) -er | there -maar | but, only -om | round, about, for etc -hem | him -dan | then -zou | should/would, past tense all persons sing. of 'zullen' -of | or, whether, if -wat | what, something, anything -mijn | possessive and noun 'mine' -men | people, 'one' -dit | this -zo | so, thus, in this way -door | through by -over | over, across -ze | she, her, they, them -zich | oneself -bij | (1) a bee, (2) by, near, at -ook | also, too -tot | till, until -je | you -mij | me -uit | out of, from -der | Old Dutch form of 'van der' still found in surnames -daar | (1) there, (2) because -haar | (1) her, their, them, (2) hair -naar | (1) unpleasant, unwell etc, (2) towards, (3) as -heb | present first person sing. of 'to have' -hoe | how, why -heeft | present third person sing. of 'to have' -hebben | 'to have' and various parts thereof -deze | this -u | you -want | (1) for, (2) mitten, (3) rigging -nog | yet, still -zal | 'shall', first and third person sing. of verb 'zullen' (will) -me | me -zij | she, they -nu | now -ge | 'thou', still used in Belgium and south Netherlands -geen | none -omdat | because -iets | something, somewhat -worden | to become, grow, get -toch | yet, still -al | all, every, each -waren | (1) 'were' (2) to wander, (3) wares, (3) -veel | much, many -meer | (1) more, (2) lake -doen | to do, to make -toen | then, when -moet | noun 'spot/mote' and present form of 'to must' -ben | (1) am, (2) 'are' in interrogative second person singular of 'to be' -zonder | without -kan | noun 'can' and present form of 'to be able' -hun | their, them -dus | so, consequently -alles | all, everything, anything -onder | under, beneath -ja | yes, of course -eens | once, one day -hier | here -wie | who -werd | imperfect third person sing. of 'become' -altijd | always -doch | yet, but etc -wordt | present third person sing. of 'become' -wezen | (1) to be, (2) 'been' as in 'been fishing', (3) orphans -kunnen | to be able -ons | us/our -zelf | self -tegen | against, towards, at -na | after, near -reeds | already -wil | (1) present tense of 'want', (2) 'will', noun, (3) fender -kon | could; past tense of 'to be able' -niets | nothing -uw | your -iemand | somebody -geweest | been; past participle of 'be' -andere | other diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_no.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_no.txt deleted file mode 100644 index a7a2c28ba..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_no.txt +++ /dev/null @@ -1,194 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/norwegian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Norwegian stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This stop word list is for the dominant bokmål dialect. Words unique - | to nynorsk are marked *. - - | Revised by Jan Bruusgaard , Jan 2005 - -og | and -i | in -jeg | I -det | it/this/that -at | to (w. inf.) -en | a/an -et | a/an -den | it/this/that -til | to -er | is/am/are -som | who/that -på | on -de | they / you(formal) -med | with -han | he -av | of -ikke | not -ikkje | not * -der | there -så | so -var | was/were -meg | me -seg | you -men | but -ett | one -har | have -om | about -vi | we -min | my -mitt | my -ha | have -hadde | had -hun | she -nå | now -over | over -da | when/as -ved | by/know -fra | from -du | you -ut | out -sin | your -dem | them -oss | us -opp | up -man | you/one -kan | can -hans | his -hvor | where -eller | or -hva | what -skal | shall/must -selv | self (reflective) -sjøl | self (reflective) -her | here -alle | all -vil | will -bli | become -ble | became -blei | became * -blitt | have become -kunne | could -inn | in -når | when -være | be -kom | come -noen | some -noe | some -ville | would -dere | you -som | who/which/that -deres | their/theirs -kun | only/just -ja | yes -etter | after -ned | down -skulle | should -denne | this -for | for/because -deg | you -si | hers/his -sine | hers/his -sitt | hers/his -mot | against -å | to -meget | much -hvorfor | why -dette | this -disse | these/those -uten | without -hvordan | how -ingen | none -din | your -ditt | your -blir | become -samme | same -hvilken | which -hvilke | which (plural) -sånn | such a -inni | inside/within -mellom | between -vår | our -hver | each -hvem | who -vors | us/ours -hvis | whose -både | both -bare | only/just -enn | than -fordi | as/because -før | before -mange | many -også | also -slik | just -vært | been -være | to be -båe | both * -begge | both -siden | since -dykk | your * -dykkar | yours * -dei | they * -deira | them * -deires | theirs * -deim | them * -di | your (fem.) * -då | as/when * -eg | I * -ein | a/an * -eit | a/an * -eitt | a/an * -elles | or * -honom | he * -hjå | at * -ho | she * -hoe | she * -henne | her -hennar | her/hers -hennes | hers -hoss | how * -hossen | how * -ikkje | not * -ingi | noone * -inkje | noone * -korleis | how * -korso | how * -kva | what/which * -kvar | where * -kvarhelst | where * -kven | who/whom * -kvi | why * -kvifor | why * -me | we * -medan | while * -mi | my * -mine | my * -mykje | much * -no | now * -nokon | some (masc./neut.) * -noka | some (fem.) * -nokor | some * -noko | some * -nokre | some * -si | his/hers * -sia | since * -sidan | since * -so | so * -somt | some * -somme | some * -um | about* -upp | up * -vere | be * -vore | was * -verte | become * -vort | become * -varte | became * -vart | became * - diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_pt.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_pt.txt deleted file mode 100644 index acfeb01af..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_pt.txt +++ /dev/null @@ -1,253 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Portuguese stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - - | The following is a ranked list (commonest to rarest) of stopwords - | deriving from a large sample of text. - - | Extra words have been added at the end. - -de | of, from -a | the; to, at; her -o | the; him -que | who, that -e | and -do | de + o -da | de + a -em | in -um | a -para | for - | é from SER -com | with -não | not, no -uma | a -os | the; them -no | em + o -se | himself etc -na | em + a -por | for -mais | more -as | the; them -dos | de + os -como | as, like -mas | but - | foi from SER -ao | a + o -ele | he -das | de + as - | tem from TER -à | a + a -seu | his -sua | her -ou | or - | ser from SER -quando | when -muito | much - | há from HAV -nos | em + os; us -já | already, now - | está from EST -eu | I -também | also -só | only, just -pelo | per + o -pela | per + a -até | up to -isso | that -ela | he -entre | between - | era from SER -depois | after -sem | without -mesmo | same -aos | a + os - | ter from TER -seus | his -quem | whom -nas | em + as -me | me -esse | that -eles | they - | estão from EST -você | you - | tinha from TER - | foram from SER -essa | that -num | em + um -nem | nor -suas | her -meu | my -às | a + as -minha | my - | têm from TER -numa | em + uma -pelos | per + os -elas | they - | havia from HAV - | seja from SER -qual | which - | será from SER -nós | we - | tenho from TER -lhe | to him, her -deles | of them -essas | those -esses | those -pelas | per + as -este | this - | fosse from SER -dele | of him - - | other words. There are many contractions such as naquele = em+aquele, - | mo = me+o, but they are rare. - | Indefinite article plural forms are also rare. - -tu | thou -te | thee -vocês | you (plural) -vos | you -lhes | to them -meus | my -minhas -teu | thy -tua -teus -tuas -nosso | our -nossa -nossos -nossas - -dela | of her -delas | of them - -esta | this -estes | these -estas | these -aquele | that -aquela | that -aqueles | those -aquelas | those -isto | this -aquilo | that - - | forms of estar, to be (not including the infinitive): -estou -está -estamos -estão -estive -esteve -estivemos -estiveram -estava -estávamos -estavam -estivera -estivéramos -esteja -estejamos -estejam -estivesse -estivéssemos -estivessem -estiver -estivermos -estiverem - - | forms of haver, to have (not including the infinitive): -hei -há -havemos -hão -houve -houvemos -houveram -houvera -houvéramos -haja -hajamos -hajam -houvesse -houvéssemos -houvessem -houver -houvermos -houverem -houverei -houverá -houveremos -houverão -houveria -houveríamos -houveriam - - | forms of ser, to be (not including the infinitive): -sou -somos -são -era -éramos -eram -fui -foi -fomos -foram -fora -fôramos -seja -sejamos -sejam -fosse -fôssemos -fossem -for -formos -forem -serei -será -seremos -serão -seria -seríamos -seriam - - | forms of ter, to have (not including the infinitive): -tenho -tem -temos -tém -tinha -tínhamos -tinham -tive -teve -tivemos -tiveram -tivera -tivéramos -tenha -tenhamos -tenham -tivesse -tivéssemos -tivessem -tiver -tivermos -tiverem -terei -terá -teremos -terão -teria -teríamos -teriam diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ro.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ro.txt deleted file mode 100644 index 4fdee90a5..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ro.txt +++ /dev/null @@ -1,233 +0,0 @@ -# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -acea -aceasta -această -aceea -acei -aceia -acel -acela -acele -acelea -acest -acesta -aceste -acestea -aceşti -aceştia -acolo -acum -ai -aia -aibă -aici -al -ăla -ale -alea -ălea -altceva -altcineva -am -ar -are -aş -aşadar -asemenea -asta -ăsta -astăzi -astea -ăstea -ăştia -asupra -aţi -au -avea -avem -aveţi -azi -bine -bucur -bună -ca -că -căci -când -care -cărei -căror -cărui -cât -câte -câţi -către -câtva -ce -cel -ceva -chiar -cînd -cine -cineva -cît -cîte -cîţi -cîtva -contra -cu -cum -cumva -curând -curînd -da -dă -dacă -dar -datorită -de -deci -deja -deoarece -departe -deşi -din -dinaintea -dintr -dintre -drept -după -ea -ei -el -ele -eram -este -eşti -eu -face -fără -fi -fie -fiecare -fii -fim -fiţi -iar -ieri -îi -îl -îmi -împotriva -în -înainte -înaintea -încât -încît -încotro -între -întrucât -întrucît -îţi -la -lângă -le -li -lîngă -lor -lui -mă -mâine -mea -mei -mele -mereu -meu -mi -mine -mult -multă -mulţi -ne -nicăieri -nici -nimeni -nişte -noastră -noastre -noi -noştri -nostru -nu -ori -oricând -oricare -oricât -orice -oricînd -oricine -oricît -oricum -oriunde -până -pe -pentru -peste -pînă -poate -pot -prea -prima -primul -prin -printr -sa -să -săi -sale -sau -său -se -şi -sînt -sîntem -sînteţi -spre -sub -sunt -suntem -sunteţi -ta -tăi -tale -tău -te -ţi -ţie -tine -toată -toate -tot -toţi -totuşi -tu -un -una -unde -undeva -unei -unele -uneori -unor -vă -vi -voastră -voastre -voi -voştri -vostru -vouă -vreo -vreun diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ru.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ru.txt deleted file mode 100644 index 55271400c..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_ru.txt +++ /dev/null @@ -1,243 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/russian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | a russian stop word list. comments begin with vertical bar. each stop - | word is at the start of a line. - - | this is a ranked list (commonest to rarest) of stopwords derived from - | a large text sample. - - | letter `ё' is translated to `е'. - -и | and -в | in/into -во | alternative form -не | not -что | what/that -он | he -на | on/onto -я | i -с | from -со | alternative form -как | how -а | milder form of `no' (but) -то | conjunction and form of `that' -все | all -она | she -так | so, thus -его | him -но | but -да | yes/and -ты | thou -к | towards, by -у | around, chez -же | intensifier particle -вы | you -за | beyond, behind -бы | conditional/subj. particle -по | up to, along -только | only -ее | her -мне | to me -было | it was -вот | here is/are, particle -от | away from -меня | me -еще | still, yet, more -нет | no, there isnt/arent -о | about -из | out of -ему | to him -теперь | now -когда | when -даже | even -ну | so, well -вдруг | suddenly -ли | interrogative particle -если | if -уже | already, but homonym of `narrower' -или | or -ни | neither -быть | to be -был | he was -него | prepositional form of его -до | up to -вас | you accusative -нибудь | indef. suffix preceded by hyphen -опять | again -уж | already, but homonym of `adder' -вам | to you -сказал | he said -ведь | particle `after all' -там | there -потом | then -себя | oneself -ничего | nothing -ей | to her -может | usually with `быть' as `maybe' -они | they -тут | here -где | where -есть | there is/are -надо | got to, must -ней | prepositional form of ей -для | for -мы | we -тебя | thee -их | them, their -чем | than -была | she was -сам | self -чтоб | in order to -без | without -будто | as if -человек | man, person, one -чего | genitive form of `what' -раз | once -тоже | also -себе | to oneself -под | beneath -жизнь | life -будет | will be -ж | short form of intensifer particle `же' -тогда | then -кто | who -этот | this -говорил | was saying -того | genitive form of `that' -потому | for that reason -этого | genitive form of `this' -какой | which -совсем | altogether -ним | prepositional form of `его', `они' -здесь | here -этом | prepositional form of `этот' -один | one -почти | almost -мой | my -тем | instrumental/dative plural of `тот', `то' -чтобы | full form of `in order that' -нее | her (acc.) -кажется | it seems -сейчас | now -были | they were -куда | where to -зачем | why -сказать | to say -всех | all (acc., gen. preposn. plural) -никогда | never -сегодня | today -можно | possible, one can -при | by -наконец | finally -два | two -об | alternative form of `о', about -другой | another -хоть | even -после | after -над | above -больше | more -тот | that one (masc.) -через | across, in -эти | these -нас | us -про | about -всего | in all, only, of all -них | prepositional form of `они' (they) -какая | which, feminine -много | lots -разве | interrogative particle -сказала | she said -три | three -эту | this, acc. fem. sing. -моя | my, feminine -впрочем | moreover, besides -хорошо | good -свою | ones own, acc. fem. sing. -этой | oblique form of `эта', fem. `this' -перед | in front of -иногда | sometimes -лучше | better -чуть | a little -том | preposn. form of `that one' -нельзя | one must not -такой | such a one -им | to them -более | more -всегда | always -конечно | of course -всю | acc. fem. sing of `all' -между | between - - - | b: some paradigms - | - | personal pronouns - | - | я меня мне мной [мною] - | ты тебя тебе тобой [тобою] - | он его ему им [него, нему, ним] - | она ее эи ею [нее, нэи, нею] - | оно его ему им [него, нему, ним] - | - | мы нас нам нами - | вы вас вам вами - | они их им ими [них, ним, ними] - | - | себя себе собой [собою] - | - | demonstrative pronouns: этот (this), тот (that) - | - | этот эта это эти - | этого эты это эти - | этого этой этого этих - | этому этой этому этим - | этим этой этим [этою] этими - | этом этой этом этих - | - | тот та то те - | того ту то те - | того той того тех - | тому той тому тем - | тем той тем [тою] теми - | том той том тех - | - | determinative pronouns - | - | (a) весь (all) - | - | весь вся все все - | всего всю все все - | всего всей всего всех - | всему всей всему всем - | всем всей всем [всею] всеми - | всем всей всем всех - | - | (b) сам (himself etc) - | - | сам сама само сами - | самого саму само самих - | самого самой самого самих - | самому самой самому самим - | самим самой самим [самою] самими - | самом самой самом самих - | - | stems of verbs `to be', `to have', `to do' and modal - | - | быть бы буд быв есть суть - | име - | дел - | мог мож мочь - | уме - | хоч хот - | долж - | можн - | нужн - | нельзя - diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_sv.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_sv.txt deleted file mode 100644 index 096f87f67..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_sv.txt +++ /dev/null @@ -1,133 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/swedish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Swedish stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This is a ranked list (commonest to rarest) of stopwords derived from - | a large text sample. - - | Swedish stop words occasionally exhibit homonym clashes. For example - | så = so, but also seed. These are indicated clearly below. - -och | and -det | it, this/that -att | to (with infinitive) -i | in, at -en | a -jag | I -hon | she -som | who, that -han | he -på | on -den | it, this/that -med | with -var | where, each -sig | him(self) etc -för | for -så | so (also: seed) -till | to -är | is -men | but -ett | a -om | if; around, about -hade | had -de | they, these/those -av | of -icke | not, no -mig | me -du | you -henne | her -då | then, when -sin | his -nu | now -har | have -inte | inte någon = no one -hans | his -honom | him -skulle | 'sake' -hennes | her -där | there -min | my -man | one (pronoun) -ej | nor -vid | at, by, on (also: vast) -kunde | could -något | some etc -från | from, off -ut | out -när | when -efter | after, behind -upp | up -vi | we -dem | them -vara | be -vad | what -över | over -än | than -dig | you -kan | can -sina | his -här | here -ha | have -mot | towards -alla | all -under | under (also: wonder) -någon | some etc -eller | or (else) -allt | all -mycket | much -sedan | since -ju | why -denna | this/that -själv | myself, yourself etc -detta | this/that -åt | to -utan | without -varit | was -hur | how -ingen | no -mitt | my -ni | you -bli | to be, become -blev | from bli -oss | us -din | thy -dessa | these/those -några | some etc -deras | their -blir | from bli -mina | my -samma | (the) same -vilken | who, that -er | you, your -sådan | such a -vår | our -blivit | from bli -dess | its -inom | within -mellan | between -sådant | such a -varför | why -varje | each -vilka | who, that -ditt | thy -vem | who -vilket | who, that -sitta | his -sådana | such a -vart | each -dina | thy -vars | whose -vårt | our -våra | our -ert | your -era | your -vilkas | whose - diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_th.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_th.txt deleted file mode 100644 index 07f0fabe6..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_th.txt +++ /dev/null @@ -1,119 +0,0 @@ -# Thai stopwords from: -# "Opinion Detection in Thai Political News Columns -# Based on Subjectivity Analysis" -# Khampol Sukhum, Supot Nitsuwat, and Choochart Haruechaiyasak -ไว้ -ไม่ -ไป -ได้ -ให้ -ใน -โดย -แห่ง -แล้ว -และ -แรก -แบบ -แต่ -เอง -เห็น -เลย -เริ่ม -เรา -เมื่อ -เพื่อ -เพราะ -เป็นการ -เป็น -เปิดเผย -เปิด -เนื่องจาก -เดียวกัน -เดียว -เช่น -เฉพาะ -เคย -เข้า -เขา -อีก -อาจ -อะไร -ออก -อย่าง -อยู่ -อยาก -หาก -หลาย -หลังจาก -หลัง -หรือ -หนึ่ง -ส่วน -ส่ง -สุด -สําหรับ -ว่า -วัน -ลง -ร่วม -ราย -รับ -ระหว่าง -รวม -ยัง -มี -มาก -มา -พร้อม -พบ -ผ่าน -ผล -บาง -น่า -นี้ -นํา -นั้น -นัก -นอกจาก -ทุก -ที่สุด -ที่ -ทําให้ -ทํา -ทาง -ทั้งนี้ -ทั้ง -ถ้า -ถูก -ถึง -ต้อง -ต่างๆ -ต่าง -ต่อ -ตาม -ตั้งแต่ -ตั้ง -ด้าน -ด้วย -ดัง -ซึ่ง -ช่วง -จึง -จาก -จัด -จะ -คือ -ความ -ครั้ง -คง -ขึ้น -ของ -ขอ -ขณะ -ก่อน -ก็ -การ -กับ -กัน -กว่า -กล่าว diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_tr.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_tr.txt deleted file mode 100644 index 84d9408d4..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/stopwords_tr.txt +++ /dev/null @@ -1,212 +0,0 @@ -# Turkish stopwords from LUCENE-559 -# merged with the list from "Information Retrieval on Turkish Texts" -# (http://www.users.muohio.edu/canf/papers/JASIST2008offPrint.pdf) -acaba -altmış -altı -ama -ancak -arada -aslında -ayrıca -bana -bazı -belki -ben -benden -beni -benim -beri -beş -bile -bin -bir -birçok -biri -birkaç -birkez -birşey -birşeyi -biz -bize -bizden -bizi -bizim -böyle -böylece -bu -buna -bunda -bundan -bunlar -bunları -bunların -bunu -bunun -burada -çok -çünkü -da -daha -dahi -de -defa -değil -diğer -diye -doksan -dokuz -dolayı -dolayısıyla -dört -edecek -eden -ederek -edilecek -ediliyor -edilmesi -ediyor -eğer -elli -en -etmesi -etti -ettiği -ettiğini -gibi -göre -halen -hangi -hatta -hem -henüz -hep -hepsi -her -herhangi -herkesin -hiç -hiçbir -için -iki -ile -ilgili -ise -işte -itibaren -itibariyle -kadar -karşın -katrilyon -kendi -kendilerine -kendini -kendisi -kendisine -kendisini -kez -ki -kim -kimden -kime -kimi -kimse -kırk -milyar -milyon -mu -mü -mı -nasıl -ne -neden -nedenle -nerde -nerede -nereye -niye -niçin -o -olan -olarak -oldu -olduğu -olduğunu -olduklarını -olmadı -olmadığı -olmak -olması -olmayan -olmaz -olsa -olsun -olup -olur -olursa -oluyor -on -ona -ondan -onlar -onlardan -onları -onların -onu -onun -otuz -oysa -öyle -pek -rağmen -sadece -sanki -sekiz -seksen -sen -senden -seni -senin -siz -sizden -sizi -sizin -şey -şeyden -şeyi -şeyler -şöyle -şu -şuna -şunda -şundan -şunları -şunu -tarafından -trilyon -tüm -üç -üzere -var -vardı -ve -veya -ya -yani -yapacak -yapılan -yapılması -yapıyor -yapmak -yaptı -yaptığı -yaptığını -yaptıkları -yedi -yerine -yetmiş -yine -yirmi -yoksa -yüz -zaten diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/userdict_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/userdict_ja.txt deleted file mode 100644 index 6f0368e4d..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/lang/userdict_ja.txt +++ /dev/null @@ -1,29 +0,0 @@ -# -# This is a sample user dictionary for Kuromoji (JapaneseTokenizer) -# -# Add entries to this file in order to override the statistical model in terms -# of segmentation, readings and part-of-speech tags. Notice that entries do -# not have weights since they are always used when found. This is by-design -# in order to maximize ease-of-use. -# -# Entries are defined using the following CSV format: -# , ... , ... , -# -# Notice that a single half-width space separates tokens and readings, and -# that the number tokens and readings must match exactly. -# -# Also notice that multiple entries with the same is undefined. -# -# Whitespace only lines are ignored. Comments are not allowed on entry lines. -# - -# Custom segmentation for kanji compounds -日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞 -関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,カスタム名詞 - -# Custom segmentation for compound katakana -トートバッグ,トート バッグ,トート バッグ,かずカナ名詞 -ショルダーバッグ,ショルダー バッグ,ショルダー バッグ,かずカナ名詞 - -# Custom reading for former sumo wrestler -朝青龍,朝青龍,アサショウリュウ,カスタム人名 diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/protwords.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/protwords.txt deleted file mode 100644 index 1dfc0abec..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/protwords.txt +++ /dev/null @@ -1,21 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# Use a protected word file to protect against the stemmer reducing two -# unrelated words to the same base word. - -# Some non-words that normally won't be encountered, -# just to test that they won't be stemmed. -dontstems -zwhacky - diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/schema.xml b/search-services/alfresco-search/src/test/resources/test-files/master/conf/schema.xml deleted file mode 100644 index 024c6ebb2..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/schema.xml +++ /dev/null @@ -1,766 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/spellings.txt b/search-services/alfresco-search/src/test/resources/test-files/master/conf/spellings.txt deleted file mode 100644 index d7ede6f56..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/spellings.txt +++ /dev/null @@ -1,2 +0,0 @@ -pizza -history \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.html b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.html deleted file mode 100644 index d8f22b44e..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.html +++ /dev/null @@ -1,156 +0,0 @@ - - - - - -
    Update the Summary and FTS Status reports
    - -
    -
    -

    Alfresco Core - Summary Report

    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -

    Alfresco Core - FTS Status Report

    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -
    - - - - -

    Other Links:

    -
    -
    Note: the following links in a new window
    - - - -
    diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.menu-bottom.html b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.menu-bottom.html deleted file mode 100644 index e69de29bb..000000000 diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.menu-top.html b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/admin-extra.menu-top.html deleted file mode 100644 index e69de29bb..000000000 diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/elevate.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/elevate.xml deleted file mode 100644 index ed2bc4c3a..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/elevate.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ca.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ca.txt deleted file mode 100644 index 307a85f91..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ca.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Set of Catalan contractions for ElisionFilter -# TODO: load this as a resource from the analyzer and sync it in build.xml -d -l -m -n -s -t diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_fr.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_fr.txt deleted file mode 100644 index f1bba51b2..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_fr.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Set of French contractions for ElisionFilter -# TODO: load this as a resource from the analyzer and sync it in build.xml -l -m -t -qu -n -s -j -d -c -jusqu -quoiqu -lorsqu -puisqu diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ga.txt deleted file mode 100644 index 9ebe7fa34..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_ga.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Set of Irish contractions for ElisionFilter -# TODO: load this as a resource from the analyzer and sync it in build.xml -d -m -b diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_it.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_it.txt deleted file mode 100644 index cac040953..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/contractions_it.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Set of Italian contractions for ElisionFilter -# TODO: load this as a resource from the analyzer and sync it in build.xml -c -l -all -dall -dell -nell -sull -coll -pell -gl -agl -dagl -degl -negl -sugl -un -m -t -s -v -d diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/hyphenations_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/hyphenations_ga.txt deleted file mode 100644 index 4d2642cc5..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/hyphenations_ga.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Set of Irish hyphenations for StopFilter -# TODO: load this as a resource from the analyzer and sync it in build.xml -h -n -t diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stemdict_nl.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stemdict_nl.txt deleted file mode 100644 index 441072971..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stemdict_nl.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Set of overrides for the dutch stemmer -# TODO: load this as a resource from the analyzer and sync it in build.xml -fiets fiets -bromfiets bromfiets -ei eier -kind kinder diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stoptags_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stoptags_ja.txt deleted file mode 100644 index 71b750845..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stoptags_ja.txt +++ /dev/null @@ -1,420 +0,0 @@ -# -# This file defines a Japanese stoptag set for JapanesePartOfSpeechStopFilter. -# -# Any token with a part-of-speech tag that exactly matches those defined in this -# file are removed from the token stream. -# -# Set your own stoptags by uncommenting the lines below. Note that comments are -# not allowed on the same line as a stoptag. See LUCENE-3745 for frequency lists, -# etc. that can be useful for building you own stoptag set. -# -# The entire possible tagset is provided below for convenience. -# -##### -# noun: unclassified nouns -#名詞 -# -# noun-common: Common nouns or nouns where the sub-classification is undefined -#名詞-一般 -# -# noun-proper: Proper nouns where the sub-classification is undefined -#名詞-固有名詞 -# -# noun-proper-misc: miscellaneous proper nouns -#名詞-固有名詞-一般 -# -# noun-proper-person: Personal names where the sub-classification is undefined -#名詞-固有名詞-人名 -# -# noun-proper-person-misc: names that cannot be divided into surname and -# given name; foreign names; names where the surname or given name is unknown. -# e.g. お市の方 -#名詞-固有名詞-人名-一般 -# -# noun-proper-person-surname: Mainly Japanese surnames. -# e.g. 山田 -#名詞-固有名詞-人名-姓 -# -# noun-proper-person-given_name: Mainly Japanese given names. -# e.g. 太郎 -#名詞-固有名詞-人名-名 -# -# noun-proper-organization: Names representing organizations. -# e.g. 通産省, NHK -#名詞-固有名詞-組織 -# -# noun-proper-place: Place names where the sub-classification is undefined -#名詞-固有名詞-地域 -# -# noun-proper-place-misc: Place names excluding countries. -# e.g. アジア, バルセロナ, 京都 -#名詞-固有名詞-地域-一般 -# -# noun-proper-place-country: Country names. -# e.g. 日本, オーストラリア -#名詞-固有名詞-地域-国 -# -# noun-pronoun: Pronouns where the sub-classification is undefined -#名詞-代名詞 -# -# noun-pronoun-misc: miscellaneous pronouns: -# e.g. それ, ここ, あいつ, あなた, あちこち, いくつ, どこか, なに, みなさん, みんな, わたくし, われわれ -#名詞-代名詞-一般 -# -# noun-pronoun-contraction: Spoken language contraction made by combining a -# pronoun and the particle 'wa'. -# e.g. ありゃ, こりゃ, こりゃあ, そりゃ, そりゃあ -#名詞-代名詞-縮約 -# -# noun-adverbial: Temporal nouns such as names of days or months that behave -# like adverbs. Nouns that represent amount or ratios and can be used adverbially, -# e.g. 金曜, 一月, 午後, 少量 -#名詞-副詞可能 -# -# noun-verbal: Nouns that take arguments with case and can appear followed by -# 'suru' and related verbs (する, できる, なさる, くださる) -# e.g. インプット, 愛着, 悪化, 悪戦苦闘, 一安心, 下取り -#名詞-サ変接続 -# -# noun-adjective-base: The base form of adjectives, words that appear before な ("na") -# e.g. 健康, 安易, 駄目, だめ -#名詞-形容動詞語幹 -# -# noun-numeric: Arabic numbers, Chinese numerals, and counters like 何 (回), 数. -# e.g. 0, 1, 2, 何, 数, 幾 -#名詞-数 -# -# noun-affix: noun affixes where the sub-classification is undefined -#名詞-非自立 -# -# noun-affix-misc: Of adnominalizers, the case-marker の ("no"), and words that -# attach to the base form of inflectional words, words that cannot be classified -# into any of the other categories below. This category includes indefinite nouns. -# e.g. あかつき, 暁, かい, 甲斐, 気, きらい, 嫌い, くせ, 癖, こと, 事, ごと, 毎, しだい, 次第, -# 順, せい, 所為, ついで, 序で, つもり, 積もり, 点, どころ, の, はず, 筈, はずみ, 弾み, -# 拍子, ふう, ふり, 振り, ほう, 方, 旨, もの, 物, 者, ゆえ, 故, ゆえん, 所以, わけ, 訳, -# わり, 割り, 割, ん-口語/, もん-口語/ -#名詞-非自立-一般 -# -# noun-affix-adverbial: noun affixes that that can behave as adverbs. -# e.g. あいだ, 間, あげく, 挙げ句, あと, 後, 余り, 以外, 以降, 以後, 以上, 以前, 一方, うえ, -# 上, うち, 内, おり, 折り, かぎり, 限り, きり, っきり, 結果, ころ, 頃, さい, 際, 最中, さなか, -# 最中, じたい, 自体, たび, 度, ため, 為, つど, 都度, とおり, 通り, とき, 時, ところ, 所, -# とたん, 途端, なか, 中, のち, 後, ばあい, 場合, 日, ぶん, 分, ほか, 他, まえ, 前, まま, -# 儘, 侭, みぎり, 矢先 -#名詞-非自立-副詞可能 -# -# noun-affix-aux: noun affixes treated as 助動詞 ("auxiliary verb") in school grammars -# with the stem よう(だ) ("you(da)"). -# e.g. よう, やう, 様 (よう) -#名詞-非自立-助動詞語幹 -# -# noun-affix-adjective-base: noun affixes that can connect to the indeclinable -# connection form な (aux "da"). -# e.g. みたい, ふう -#名詞-非自立-形容動詞語幹 -# -# noun-special: special nouns where the sub-classification is undefined. -#名詞-特殊 -# -# noun-special-aux: The そうだ ("souda") stem form that is used for reporting news, is -# treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the base -# form of inflectional words. -# e.g. そう -#名詞-特殊-助動詞語幹 -# -# noun-suffix: noun suffixes where the sub-classification is undefined. -#名詞-接尾 -# -# noun-suffix-misc: Of the nouns or stem forms of other parts of speech that connect -# to ガル or タイ and can combine into compound nouns, words that cannot be classified into -# any of the other categories below. In general, this category is more inclusive than -# 接尾語 ("suffix") and is usually the last element in a compound noun. -# e.g. おき, かた, 方, 甲斐 (がい), がかり, ぎみ, 気味, ぐるみ, (～した) さ, 次第, 済 (ず) み, -# よう, (でき)っこ, 感, 観, 性, 学, 類, 面, 用 -#名詞-接尾-一般 -# -# noun-suffix-person: Suffixes that form nouns and attach to person names more often -# than other nouns. -# e.g. 君, 様, 著 -#名詞-接尾-人名 -# -# noun-suffix-place: Suffixes that form nouns and attach to place names more often -# than other nouns. -# e.g. 町, 市, 県 -#名詞-接尾-地域 -# -# noun-suffix-verbal: Of the suffixes that attach to nouns and form nouns, those that -# can appear before スル ("suru"). -# e.g. 化, 視, 分け, 入り, 落ち, 買い -#名詞-接尾-サ変接続 -# -# noun-suffix-aux: The stem form of そうだ (様態) that is used to indicate conditions, -# is treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the -# conjunctive form of inflectional words. -# e.g. そう -#名詞-接尾-助動詞語幹 -# -# noun-suffix-adjective-base: Suffixes that attach to other nouns or the conjunctive -# form of inflectional words and appear before the copula だ ("da"). -# e.g. 的, げ, がち -#名詞-接尾-形容動詞語幹 -# -# noun-suffix-adverbial: Suffixes that attach to other nouns and can behave as adverbs. -# e.g. 後 (ご), 以後, 以降, 以前, 前後, 中, 末, 上, 時 (じ) -#名詞-接尾-副詞可能 -# -# noun-suffix-classifier: Suffixes that attach to numbers and form nouns. This category -# is more inclusive than 助数詞 ("classifier") and includes common nouns that attach -# to numbers. -# e.g. 個, つ, 本, 冊, パーセント, cm, kg, カ月, か国, 区画, 時間, 時半 -#名詞-接尾-助数詞 -# -# noun-suffix-special: Special suffixes that mainly attach to inflecting words. -# e.g. (楽し) さ, (考え) 方 -#名詞-接尾-特殊 -# -# noun-suffix-conjunctive: Nouns that behave like conjunctions and join two words -# together. -# e.g. (日本) 対 (アメリカ), 対 (アメリカ), (3) 対 (5), (女優) 兼 (主婦) -#名詞-接続詞的 -# -# noun-verbal_aux: Nouns that attach to the conjunctive particle て ("te") and are -# semantically verb-like. -# e.g. ごらん, ご覧, 御覧, 頂戴 -#名詞-動詞非自立的 -# -# noun-quotation: text that cannot be segmented into words, proverbs, Chinese poetry, -# dialects, English, etc. Currently, the only entry for 名詞 引用文字列 ("noun quotation") -# is いわく ("iwaku"). -#名詞-引用文字列 -# -# noun-nai_adjective: Words that appear before the auxiliary verb ない ("nai") and -# behave like an adjective. -# e.g. 申し訳, 仕方, とんでも, 違い -#名詞-ナイ形容詞語幹 -# -##### -# prefix: unclassified prefixes -#接頭詞 -# -# prefix-nominal: Prefixes that attach to nouns (including adjective stem forms) -# excluding numerical expressions. -# e.g. お (水), 某 (氏), 同 (社), 故 (～氏), 高 (品質), お (見事), ご (立派) -#接頭詞-名詞接続 -# -# prefix-verbal: Prefixes that attach to the imperative form of a verb or a verb -# in conjunctive form followed by なる/なさる/くださる. -# e.g. お (読みなさい), お (座り) -#接頭詞-動詞接続 -# -# prefix-adjectival: Prefixes that attach to adjectives. -# e.g. お (寒いですねえ), バカ (でかい) -#接頭詞-形容詞接続 -# -# prefix-numerical: Prefixes that attach to numerical expressions. -# e.g. 約, およそ, 毎時 -#接頭詞-数接続 -# -##### -# verb: unclassified verbs -#動詞 -# -# verb-main: -#動詞-自立 -# -# verb-auxiliary: -#動詞-非自立 -# -# verb-suffix: -#動詞-接尾 -# -##### -# adjective: unclassified adjectives -#形容詞 -# -# adjective-main: -#形容詞-自立 -# -# adjective-auxiliary: -#形容詞-非自立 -# -# adjective-suffix: -#形容詞-接尾 -# -##### -# adverb: unclassified adverbs -#副詞 -# -# adverb-misc: Words that can be segmented into one unit and where adnominal -# modification is not possible. -# e.g. あいかわらず, 多分 -#副詞-一般 -# -# adverb-particle_conjunction: Adverbs that can be followed by の, は, に, -# な, する, だ, etc. -# e.g. こんなに, そんなに, あんなに, なにか, なんでも -#副詞-助詞類接続 -# -##### -# adnominal: Words that only have noun-modifying forms. -# e.g. この, その, あの, どの, いわゆる, なんらかの, 何らかの, いろんな, こういう, そういう, ああいう, -# どういう, こんな, そんな, あんな, どんな, 大きな, 小さな, おかしな, ほんの, たいした, -# 「(, も) さる (ことながら)」, 微々たる, 堂々たる, 単なる, いかなる, 我が」「同じ, 亡き -#連体詞 -# -##### -# conjunction: Conjunctions that can occur independently. -# e.g. が, けれども, そして, じゃあ, それどころか -接続詞 -# -##### -# particle: unclassified particles. -助詞 -# -# particle-case: case particles where the subclassification is undefined. -助詞-格助詞 -# -# particle-case-misc: Case particles. -# e.g. から, が, で, と, に, へ, より, を, の, にて -助詞-格助詞-一般 -# -# particle-case-quote: the "to" that appears after nouns, a person’s speech, -# quotation marks, expressions of decisions from a meeting, reasons, judgements, -# conjectures, etc. -# e.g. ( だ) と (述べた.), ( である) と (して執行猶予...) -助詞-格助詞-引用 -# -# particle-case-compound: Compounds of particles and verbs that mainly behave -# like case particles. -# e.g. という, といった, とかいう, として, とともに, と共に, でもって, にあたって, に当たって, に当って, -# にあたり, に当たり, に当り, に当たる, にあたる, において, に於いて,に於て, における, に於ける, -# にかけ, にかけて, にかんし, に関し, にかんして, に関して, にかんする, に関する, に際し, -# に際して, にしたがい, に従い, に従う, にしたがって, に従って, にたいし, に対し, にたいして, -# に対して, にたいする, に対する, について, につき, につけ, につけて, につれ, につれて, にとって, -# にとり, にまつわる, によって, に依って, に因って, により, に依り, に因り, による, に依る, に因る, -# にわたって, にわたる, をもって, を以って, を通じ, を通じて, を通して, をめぐって, をめぐり, をめぐる, -# って-口語/, ちゅう-関西弁「という」/, (何) ていう (人)-口語/, っていう-口語/, といふ, とかいふ -助詞-格助詞-連語 -# -# particle-conjunctive: -# e.g. から, からには, が, けれど, けれども, けど, し, つつ, て, で, と, ところが, どころか, とも, ども, -# ながら, なり, ので, のに, ば, ものの, や ( した), やいなや, (ころん) じゃ(いけない)-口語/, -# (行っ) ちゃ(いけない)-口語/, (言っ) たって (しかたがない)-口語/, (それがなく)ったって (平気)-口語/ -助詞-接続助詞 -# -# particle-dependency: -# e.g. こそ, さえ, しか, すら, は, も, ぞ -助詞-係助詞 -# -# particle-adverbial: -# e.g. がてら, かも, くらい, 位, ぐらい, しも, (学校) じゃ(これが流行っている)-口語/, -# (それ)じゃあ (よくない)-口語/, ずつ, (私) なぞ, など, (私) なり (に), (先生) なんか (大嫌い)-口語/, -# (私) なんぞ, (先生) なんて (大嫌い)-口語/, のみ, だけ, (私) だって-口語/, だに, -# (彼)ったら-口語/, (お茶) でも (いかが), 等 (とう), (今後) とも, ばかり, ばっか-口語/, ばっかり-口語/, -# ほど, 程, まで, 迄, (誰) も (が)([助詞-格助詞] および [助詞-係助詞] の前に位置する「も」) -助詞-副助詞 -# -# particle-interjective: particles with interjective grammatical roles. -# e.g. (松島) や -助詞-間投助詞 -# -# particle-coordinate: -# e.g. と, たり, だの, だり, とか, なり, や, やら -助詞-並立助詞 -# -# particle-final: -# e.g. かい, かしら, さ, ぜ, (だ)っけ-口語/, (とまってる) で-方言/, な, ナ, なあ-口語/, ぞ, ね, ネ, -# ねぇ-口語/, ねえ-口語/, ねん-方言/, の, のう-口語/, や, よ, ヨ, よぉ-口語/, わ, わい-口語/ -助詞-終助詞 -# -# particle-adverbial/conjunctive/final: The particle "ka" when unknown whether it is -# adverbial, conjunctive, or sentence final. For example: -# (a) 「A か B か」. Ex:「(国内で運用する) か,(海外で運用する) か (.)」 -# (b) Inside an adverb phrase. Ex:「(幸いという) か (, 死者はいなかった.)」 -# 「(祈りが届いたせい) か (, 試験に合格した.)」 -# (c) 「かのように」. Ex:「(何もなかった) か (のように振る舞った.)」 -# e.g. か -助詞-副助詞／並立助詞／終助詞 -# -# particle-adnominalizer: The "no" that attaches to nouns and modifies -# non-inflectional words. -助詞-連体化 -# -# particle-adnominalizer: The "ni" and "to" that appear following nouns and adverbs -# that are giongo, giseigo, or gitaigo. -# e.g. に, と -助詞-副詞化 -# -# particle-special: A particle that does not fit into one of the above classifications. -# This includes particles that are used in Tanka, Haiku, and other poetry. -# e.g. かな, けむ, ( しただろう) に, (あんた) にゃ(わからん), (俺) ん (家) -助詞-特殊 -# -##### -# auxiliary-verb: -助動詞 -# -##### -# interjection: Greetings and other exclamations. -# e.g. おはよう, おはようございます, こんにちは, こんばんは, ありがとう, どうもありがとう, ありがとうございます, -# いただきます, ごちそうさま, さよなら, さようなら, はい, いいえ, ごめん, ごめんなさい -#感動詞 -# -##### -# symbol: unclassified Symbols. -記号 -# -# symbol-misc: A general symbol not in one of the categories below. -# e.g. [○◎@$〒→+] -記号-一般 -# -# symbol-comma: Commas -# e.g. [,、] -記号-読点 -# -# symbol-period: Periods and full stops. -# e.g. [.．。] -記号-句点 -# -# symbol-space: Full-width whitespace. -記号-空白 -# -# symbol-open_bracket: -# e.g. [({‘“『【] -記号-括弧開 -# -# symbol-close_bracket: -# e.g. [)}’”』」】] -記号-括弧閉 -# -# symbol-alphabetic: -#記号-アルファベット -# -##### -# other: unclassified other -#その他 -# -# other-interjection: Words that are hard to classify as noun-suffixes or -# sentence-final particles. -# e.g. (だ)ァ -その他-間投 -# -##### -# filler: Aizuchi that occurs during a conversation or sounds inserted as filler. -# e.g. あの, うんと, えと -フィラー -# -##### -# non-verbal: non-verbal sound. -非言語音 -# -##### -# fragment: -#語断片 -# -##### -# unknown: unknown part of speech. -#未知語 -# -##### End of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ar.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ar.txt deleted file mode 100644 index 046829db6..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ar.txt +++ /dev/null @@ -1,125 +0,0 @@ -# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -# Cleaned on October 11, 2009 (not normalized, so use before normalization) -# This means that when modifying this list, you might need to add some -# redundant entries, for example containing forms with both أ and ا -من -ومن -منها -منه -في -وفي -فيها -فيه -و -ف -ثم -او -أو -ب -بها -به -ا -أ -اى -اي -أي -أى -لا -ولا -الا -ألا -إلا -لكن -ما -وما -كما -فما -عن -مع -اذا -إذا -ان -أن -إن -انها -أنها -إنها -انه -أنه -إنه -بان -بأن -فان -فأن -وان -وأن -وإن -التى -التي -الذى -الذي -الذين -الى -الي -إلى -إلي -على -عليها -عليه -اما -أما -إما -ايضا -أيضا -كل -وكل -لم -ولم -لن -ولن -هى -هي -هو -وهى -وهي -وهو -فهى -فهي -فهو -انت -أنت -لك -لها -له -هذه -هذا -تلك -ذلك -هناك -كانت -كان -يكون -تكون -وكانت -وكان -غير -بعض -قد -نحو -بين -بينما -منذ -ضمن -حيث -الان -الآن -خلال -بعد -قبل -حتى -عند -عندما -لدى -جميع diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_bg.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_bg.txt deleted file mode 100644 index 1ae4ba2ae..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_bg.txt +++ /dev/null @@ -1,193 +0,0 @@ -# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -а -аз -ако -ала -бе -без -беше -би -бил -била -били -било -близо -бъдат -бъде -бяха -в -вас -ваш -ваша -вероятно -вече -взема -ви -вие -винаги -все -всеки -всички -всичко -всяка -във -въпреки -върху -г -ги -главно -го -д -да -дали -до -докато -докога -дори -досега -доста -е -едва -един -ето -за -зад -заедно -заради -засега -затова -защо -защото -и -из -или -им -има -имат -иска -й -каза -как -каква -какво -както -какъв -като -кога -когато -което -които -кой -който -колко -която -къде -където -към -ли -м -ме -между -мен -ми -мнозина -мога -могат -може -моля -момента -му -н -на -над -назад -най -направи -напред -например -нас -не -него -нея -ни -ние -никой -нито -но -някои -някой -няма -обаче -около -освен -особено -от -отгоре -отново -още -пак -по -повече -повечето -под -поне -поради -после -почти -прави -пред -преди -през -при -пък -първо -с -са -само -се -сега -си -скоро -след -сме -според -сред -срещу -сте -съм -със -също -т -тази -така -такива -такъв -там -твой -те -тези -ти -тн -то -това -тогава -този -той -толкова -точно -трябва -тук -тъй -тя -тях -у -харесва -ч -че -често -чрез -ще -щом -я diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ca.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ca.txt deleted file mode 100644 index 3da65deaf..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ca.txt +++ /dev/null @@ -1,220 +0,0 @@ -# Catalan stopwords from http://github.com/vcl/cue.language (Apache 2 Licensed) -a -abans -ací -ah -així -això -al -als -aleshores -algun -alguna -algunes -alguns -alhora -allà -allí -allò -altra -altre -altres -amb -ambdós -ambdues -apa -aquell -aquella -aquelles -aquells -aquest -aquesta -aquestes -aquests -aquí -baix -cada -cadascú -cadascuna -cadascunes -cadascuns -com -contra -d'un -d'una -d'unes -d'uns -dalt -de -del -dels -des -després -dins -dintre -donat -doncs -durant -e -eh -el -els -em -en -encara -ens -entre -érem -eren -éreu -es -és -esta -està -estàvem -estaven -estàveu -esteu -et -etc -ets -fins -fora -gairebé -ha -han -has -havia -he -hem -heu -hi -ho -i -igual -iguals -ja -l'hi -la -les -li -li'n -llavors -m'he -ma -mal -malgrat -mateix -mateixa -mateixes -mateixos -me -mentre -més -meu -meus -meva -meves -molt -molta -moltes -molts -mon -mons -n'he -n'hi -ne -ni -no -nogensmenys -només -nosaltres -nostra -nostre -nostres -o -oh -oi -on -pas -pel -pels -per -però -perquè -poc -poca -pocs -poques -potser -propi -qual -quals -quan -quant -que -què -quelcom -qui -quin -quina -quines -quins -s'ha -s'han -sa -semblant -semblants -ses -seu -seus -seva -seva -seves -si -sobre -sobretot -sóc -solament -sols -son -són -sons -sota -sou -t'ha -t'han -t'he -ta -tal -també -tampoc -tan -tant -tanta -tantes -teu -teus -teva -teves -ton -tons -tot -tota -totes -tots -un -una -unes -uns -us -va -vaig -vam -van -vas -veu -vosaltres -vostra -vostre -vostres diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ckb.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ckb.txt deleted file mode 100644 index 87abf118f..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ckb.txt +++ /dev/null @@ -1,136 +0,0 @@ -# set of kurdish stopwords -# note these have been normalized with our scheme (e represented with U+06D5, etc) -# constructed from: -# * Fig 5 of "Building A Test Collection For Sorani Kurdish" (Esmaili et al) -# * "Sorani Kurdish: A Reference Grammar with selected readings" (Thackston) -# * Corpus-based analysis of 77M word Sorani collection: wikipedia, news, blogs, etc - -# and -و -# which -کە -# of -ی -# made/did -کرد -# that/which -ئەوەی -# on/head -سەر -# two -دوو -# also -هەروەها -# from/that -لەو -# makes/does -دەکات -# some -چەند -# every -هەر - -# demonstratives -# that -ئەو -# this -ئەم - -# personal pronouns -# I -من -# we -ئێمە -# you -تۆ -# you -ئێوە -# he/she/it -ئەو -# they -ئەوان - -# prepositions -# to/with/by -بە -پێ -# without -بەبێ -# along with/while/during -بەدەم -# in the opinion of -بەلای -# according to -بەپێی -# before -بەرلە -# in the direction of -بەرەوی -# in front of/toward -بەرەوە -# before/in the face of -بەردەم -# without -بێ -# except for -بێجگە -# for -بۆ -# on/in -دە -تێ -# with -دەگەڵ -# after -دوای -# except for/aside from -جگە -# in/from -لە -لێ -# in front of/before/because of -لەبەر -# between/among -لەبەینی -# concerning/about -لەبابەت -# concerning -لەبارەی -# instead of -لەباتی -# beside -لەبن -# instead of -لەبرێتی -# behind -لەدەم -# with/together with -لەگەڵ -# by -لەلایەن -# within -لەناو -# between/among -لەنێو -# for the sake of -لەپێناوی -# with respect to -لەرەوی -# by means of/for -لەرێ -# for the sake of -لەرێگا -# on/on top of/according to -لەسەر -# under -لەژێر -# between/among -ناو -# between/among -نێوان -# after -پاش -# before -پێش -# like -وەک diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_cz.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_cz.txt deleted file mode 100644 index 53c6097da..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_cz.txt +++ /dev/null @@ -1,172 +0,0 @@ -a -s -k -o -i -u -v -z -dnes -cz -tímto -budeš -budem -byli -jseš -můj -svým -ta -tomto -tohle -tuto -tyto -jej -zda -proč -máte -tato -kam -tohoto -kdo -kteří -mi -nám -tom -tomuto -mít -nic -proto -kterou -byla -toho -protože -asi -ho -naši -napište -re -což -tím -takže -svých -její -svými -jste -aj -tu -tedy -teto -bylo -kde -ke -pravé -ji -nad -nejsou -či -pod -téma -mezi -přes -ty -pak -vám -ani -když -však -neg -jsem -tento -článku -články -aby -jsme -před -pta -jejich -byl -ještě -až -bez -také -pouze -první -vaše -která -nás -nový -tipy -pokud -může -strana -jeho -své -jiné -zprávy -nové -není -vás -jen -podle -zde -už -být -více -bude -již -než -který -by -které -co -nebo -ten -tak -má -při -od -po -jsou -jak -další -ale -si -se -ve -to -jako -za -zpět -ze -do -pro -je -na -atd -atp -jakmile -přičemž -já -on -ona -ono -oni -ony -my -vy -jí -ji -mě -mne -jemu -tomu -těm -těmu -němu -němuž -jehož -jíž -jelikož -jež -jakož -načež diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_da.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_da.txt deleted file mode 100644 index 42e6145b9..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_da.txt +++ /dev/null @@ -1,110 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/danish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Danish stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This is a ranked list (commonest to rarest) of stopwords derived from - | a large text sample. - - -og | and -i | in -jeg | I -det | that (dem. pronoun)/it (pers. pronoun) -at | that (in front of a sentence)/to (with infinitive) -en | a/an -den | it (pers. pronoun)/that (dem. pronoun) -til | to/at/for/until/against/by/of/into, more -er | present tense of "to be" -som | who, as -på | on/upon/in/on/at/to/after/of/with/for, on -de | they -med | with/by/in, along -han | he -af | of/by/from/off/for/in/with/on, off -for | at/for/to/from/by/of/ago, in front/before, because -ikke | not -der | who/which, there/those -var | past tense of "to be" -mig | me/myself -sig | oneself/himself/herself/itself/themselves -men | but -et | a/an/one, one (number), someone/somebody/one -har | present tense of "to have" -om | round/about/for/in/a, about/around/down, if -vi | we -min | my -havde | past tense of "to have" -ham | him -hun | she -nu | now -over | over/above/across/by/beyond/past/on/about, over/past -da | then, when/as/since -fra | from/off/since, off, since -du | you -ud | out -sin | his/her/its/one's -dem | them -os | us/ourselves -op | up -man | you/one -hans | his -hvor | where -eller | or -hvad | what -skal | must/shall etc. -selv | myself/youself/herself/ourselves etc., even -her | here -alle | all/everyone/everybody etc. -vil | will (verb) -blev | past tense of "to stay/to remain/to get/to become" -kunne | could -ind | in -når | when -være | present tense of "to be" -dog | however/yet/after all -noget | something -ville | would -jo | you know/you see (adv), yes -deres | their/theirs -efter | after/behind/according to/for/by/from, later/afterwards -ned | down -skulle | should -denne | this -end | than -dette | this -mit | my/mine -også | also -under | under/beneath/below/during, below/underneath -have | have -dig | you -anden | other -hende | her -mine | my -alt | everything -meget | much/very, plenty of -sit | his, her, its, one's -sine | his, her, its, one's -vor | our -mod | against -disse | these -hvis | if -din | your/yours -nogle | some -hos | by/at -blive | be/become -mange | many -ad | by/through -bliver | present tense of "to be/to become" -hendes | her/hers -været | be -thi | for (conj) -jer | you -sådan | such, like this/like that diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_de.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_de.txt deleted file mode 100644 index 86525e7ae..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_de.txt +++ /dev/null @@ -1,294 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A German stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | The number of forms in this list is reduced significantly by passing it - | through the German stemmer. - - -aber | but - -alle | all -allem -allen -aller -alles - -als | than, as -also | so -am | an + dem -an | at - -ander | other -andere -anderem -anderen -anderer -anderes -anderm -andern -anderr -anders - -auch | also -auf | on -aus | out of -bei | by -bin | am -bis | until -bist | art -da | there -damit | with it -dann | then - -der | the -den -des -dem -die -das - -daß | that - -derselbe | the same -derselben -denselben -desselben -demselben -dieselbe -dieselben -dasselbe - -dazu | to that - -dein | thy -deine -deinem -deinen -deiner -deines - -denn | because - -derer | of those -dessen | of him - -dich | thee -dir | to thee -du | thou - -dies | this -diese -diesem -diesen -dieser -dieses - - -doch | (several meanings) -dort | (over) there - - -durch | through - -ein | a -eine -einem -einen -einer -eines - -einig | some -einige -einigem -einigen -einiger -einiges - -einmal | once - -er | he -ihn | him -ihm | to him - -es | it -etwas | something - -euer | your -eure -eurem -euren -eurer -eures - -für | for -gegen | towards -gewesen | p.p. of sein -hab | have -habe | have -haben | have -hat | has -hatte | had -hatten | had -hier | here -hin | there -hinter | behind - -ich | I -mich | me -mir | to me - - -ihr | you, to her -ihre -ihrem -ihren -ihrer -ihres -euch | to you - -im | in + dem -in | in -indem | while -ins | in + das -ist | is - -jede | each, every -jedem -jeden -jeder -jedes - -jene | that -jenem -jenen -jener -jenes - -jetzt | now -kann | can - -kein | no -keine -keinem -keinen -keiner -keines - -können | can -könnte | could -machen | do -man | one - -manche | some, many a -manchem -manchen -mancher -manches - -mein | my -meine -meinem -meinen -meiner -meines - -mit | with -muss | must -musste | had to -nach | to(wards) -nicht | not -nichts | nothing -noch | still, yet -nun | now -nur | only -ob | whether -oder | or -ohne | without -sehr | very - -sein | his -seine -seinem -seinen -seiner -seines - -selbst | self -sich | herself - -sie | they, she -ihnen | to them - -sind | are -so | so - -solche | such -solchem -solchen -solcher -solches - -soll | shall -sollte | should -sondern | but -sonst | else -über | over -um | about, around -und | and - -uns | us -unse -unsem -unsen -unser -unses - -unter | under -viel | much -vom | von + dem -von | from -vor | before -während | while -war | was -waren | were -warst | wast -was | what -weg | away, off -weil | because -weiter | further - -welche | which -welchem -welchen -welcher -welches - -wenn | when -werde | will -werden | will -wie | how -wieder | again -will | want -wir | we -wird | will -wirst | willst -wo | where -wollen | want -wollte | wanted -würde | would -würden | would -zu | to -zum | zu + dem -zur | zu + der -zwar | indeed -zwischen | between - diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_el.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_el.txt deleted file mode 100644 index 232681f5b..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_el.txt +++ /dev/null @@ -1,78 +0,0 @@ -# Lucene Greek Stopwords list -# Note: by default this file is used after GreekLowerCaseFilter, -# so when modifying this file use 'σ' instead of 'ς' -ο -η -το -οι -τα -του -τησ -των -τον -την -και -κι -κ -ειμαι -εισαι -ειναι -ειμαστε -ειστε -στο -στον -στη -στην -μα -αλλα -απο -για -προσ -με -σε -ωσ -παρα -αντι -κατα -μετα -θα -να -δε -δεν -μη -μην -επι -ενω -εαν -αν -τοτε -που -πωσ -ποιοσ -ποια -ποιο -ποιοι -ποιεσ -ποιων -ποιουσ -αυτοσ -αυτη -αυτο -αυτοι -αυτων -αυτουσ -αυτεσ -αυτα -εκεινοσ -εκεινη -εκεινο -εκεινοι -εκεινεσ -εκεινα -εκεινων -εκεινουσ -οπωσ -ομωσ -ισωσ -οσο -οτι diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_en.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_en.txt deleted file mode 100644 index 8bb2b7de4..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_en.txt +++ /dev/null @@ -1,332 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -### Top 200 words based on frequency in common use + stemmed versions -### > 1000 per million words -the -of -and -a -in -to -it -is -was -wa -I -for -that -you -he -be -with -on -by -at -have -are -ar -not -this -thi -but -had -they -thei -his -hi -from - -### > 3000 PMW -she -which -or -we -an -were -as - -### > 2000 PMW -do -been -their -has -ha -would -there -what -will -all -if -can -her -#said -who - -### Top 50 -### > 1000 PMW -#one -#on -so -up -them -some -when -could -him -into -its -it -then -#two -out -#time -my -about -#did -#your -#now -me -no -other -only -onli -#just -more -these -also -#people -#peopl -#know -any -ani -#first -#see -very -veri -new -#may -#mai -#well -should -#like -than -how - -### Top 90 -### > 900 PMW -#get -#way -#wai -our -#made -#got -after -#think -between -#many -#mani -#years -#year - -### Top 100 -### > 800 PMW -er -those -#go -being -be -because -becaus -down -#yeah - -### > 700 PMW -#three -#good -#back -#make -such -#through -#year -#over -#must -#still -#even -#take -#too - -### Top 120 above - -### > 600 PMW -#here -#come -#own -#last -#does -#doe -#oh -#say -#sai -#work -#where -#erm -#us -#government -#govern -#same -#man -#might -#day -#dai -#yes -#ye -#however -#howev - -### > 500 PMW -#put -#world -#another -#anoth -#want -#life -#most -#against -#again -#never -#under -#old -#much -#something -#someth -#Mr -#why -#each -#while -#house -#hous - -### > 400 PMW -#part -#number -#out of -#found -#off -#different -#differ -#went -#really -#realli -#thought -#came -#used -#us -#children -#always -#alwai -#four -#without -#give -#few -#within -#system -#local -#place -#great -#during -#dure -#although -#small -#before -#befor -#look -#case -#next -#end -#things -#thing -#social -#find -#group -#quite -#quit -#mean -#five -#party -#parti -#company -#compani -#every -#everi - -### > 300 PMW -#women -#says -#sai -#important -#import -#took - -### Top 200 - - - - - - -###################### Lucene defaults .... - - -# Standard english stop words taken from Lucene's StopAnalyzer -# - Added stemmed varients - are -> ar, they -> thei, this -> thi, was -> wa -#a -#an -#and -#are -#ar -#as -#at -#be -#but -#by -#for -#if -#in -#into -#is -#it -#no -#not -#of -#on -#or -#such -#that -#the -#their -#then -#there -#these -#they -#thei -#this -#thi -#to -#was -#wa -#will -#with diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_es.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_es.txt deleted file mode 100644 index 487d78c8d..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_es.txt +++ /dev/null @@ -1,356 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Spanish stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - - | The following is a ranked list (commonest to rarest) of stopwords - | deriving from a large sample of text. - - | Extra words have been added at the end. - -de | from, of -la | the, her -que | who, that -el | the -en | in -y | and -a | to -los | the, them -del | de + el -se | himself, from him etc -las | the, them -por | for, by, etc -un | a -para | for -con | with -no | no -una | a -su | his, her -al | a + el - | es from SER -lo | him -como | how -más | more -pero | pero -sus | su plural -le | to him, her -ya | already -o | or - | fue from SER -este | this - | ha from HABER -sí | himself etc -porque | because -esta | this - | son from SER -entre | between - | está from ESTAR -cuando | when -muy | very -sin | without -sobre | on - | ser from SER - | tiene from TENER -también | also -me | me -hasta | until -hay | there is/are -donde | where - | han from HABER -quien | whom, that - | están from ESTAR - | estado from ESTAR -desde | from -todo | all -nos | us -durante | during - | estados from ESTAR -todos | all -uno | a -les | to them -ni | nor -contra | against -otros | other - | fueron from SER -ese | that -eso | that - | había from HABER -ante | before -ellos | they -e | and (variant of y) -esto | this -mí | me -antes | before -algunos | some -qué | what? -unos | a -yo | I -otro | other -otras | other -otra | other -él | he -tanto | so much, many -esa | that -estos | these -mucho | much, many -quienes | who -nada | nothing -muchos | many -cual | who - | sea from SER -poco | few -ella | she -estar | to be - | haber from HABER -estas | these - | estaba from ESTAR - | estamos from ESTAR -algunas | some -algo | something -nosotros | we - - | other forms - -mi | me -mis | mi plural -tú | thou -te | thee -ti | thee -tu | thy -tus | tu plural -ellas | they -nosotras | we -vosotros | you -vosotras | you -os | you -mío | mine -mía | -míos | -mías | -tuyo | thine -tuya | -tuyos | -tuyas | -suyo | his, hers, theirs -suya | -suyos | -suyas | -nuestro | ours -nuestra | -nuestros | -nuestras | -vuestro | yours -vuestra | -vuestros | -vuestras | -esos | those -esas | those - - | forms of estar, to be (not including the infinitive): -estoy -estás -está -estamos -estáis -están -esté -estés -estemos -estéis -estén -estaré -estarás -estará -estaremos -estaréis -estarán -estaría -estarías -estaríamos -estaríais -estarían -estaba -estabas -estábamos -estabais -estaban -estuve -estuviste -estuvo -estuvimos -estuvisteis -estuvieron -estuviera -estuvieras -estuviéramos -estuvierais -estuvieran -estuviese -estuvieses -estuviésemos -estuvieseis -estuviesen -estando -estado -estada -estados -estadas -estad - - | forms of haber, to have (not including the infinitive): -he -has -ha -hemos -habéis -han -haya -hayas -hayamos -hayáis -hayan -habré -habrás -habrá -habremos -habréis -habrán -habría -habrías -habríamos -habríais -habrían -había -habías -habíamos -habíais -habían -hube -hubiste -hubo -hubimos -hubisteis -hubieron -hubiera -hubieras -hubiéramos -hubierais -hubieran -hubiese -hubieses -hubiésemos -hubieseis -hubiesen -habiendo -habido -habida -habidos -habidas - - | forms of ser, to be (not including the infinitive): -soy -eres -es -somos -sois -son -sea -seas -seamos -seáis -sean -seré -serás -será -seremos -seréis -serán -sería -serías -seríamos -seríais -serían -era -eras -éramos -erais -eran -fui -fuiste -fue -fuimos -fuisteis -fueron -fuera -fueras -fuéramos -fuerais -fueran -fuese -fueses -fuésemos -fueseis -fuesen -siendo -sido - | sed also means 'thirst' - - | forms of tener, to have (not including the infinitive): -tengo -tienes -tiene -tenemos -tenéis -tienen -tenga -tengas -tengamos -tengáis -tengan -tendré -tendrás -tendrá -tendremos -tendréis -tendrán -tendría -tendrías -tendríamos -tendríais -tendrían -tenía -tenías -teníamos -teníais -tenían -tuve -tuviste -tuvo -tuvimos -tuvisteis -tuvieron -tuviera -tuvieras -tuviéramos -tuvierais -tuvieran -tuviese -tuvieses -tuviésemos -tuvieseis -tuviesen -teniendo -tenido -tenida -tenidos -tenidas -tened - diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_eu.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_eu.txt deleted file mode 100644 index 25f1db934..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_eu.txt +++ /dev/null @@ -1,99 +0,0 @@ -# example set of basque stopwords -al -anitz -arabera -asko -baina -bat -batean -batek -bati -batzuei -batzuek -batzuetan -batzuk -bera -beraiek -berau -berauek -bere -berori -beroriek -beste -bezala -da -dago -dira -ditu -du -dute -edo -egin -ere -eta -eurak -ez -gainera -gu -gutxi -guzti -haiei -haiek -haietan -hainbeste -hala -han -handik -hango -hara -hari -hark -hartan -hau -hauei -hauek -hauetan -hemen -hemendik -hemengo -hi -hona -honek -honela -honetan -honi -hor -hori -horiei -horiek -horietan -horko -horra -horrek -horrela -horretan -horri -hortik -hura -izan -ni -noiz -nola -non -nondik -nongo -nor -nora -ze -zein -zen -zenbait -zenbat -zer -zergatik -ziren -zituen -zu -zuek -zuen -zuten diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fa.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fa.txt deleted file mode 100644 index 723641c6d..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fa.txt +++ /dev/null @@ -1,313 +0,0 @@ -# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -# Note: by default this file is used after normalization, so when adding entries -# to this file, use the arabic 'ي' instead of 'ی' -انان -نداشته -سراسر -خياه -ايشان -وي -تاكنون -بيشتري -دوم -پس -ناشي -وگو -يا -داشتند -سپس -هنگام -هرگز -پنج -نشان -امسال -ديگر -گروهي -شدند -چطور -ده -و -دو -نخستين -ولي -چرا -چه -وسط -ه -كدام -قابل -يك -رفت -هفت -همچنين -در -هزار -بله -بلي -شايد -اما -شناسي -گرفته -دهد -داشته -دانست -داشتن -خواهيم -ميليارد -وقتيكه -امد -خواهد -جز -اورده -شده -بلكه -خدمات -شدن -برخي -نبود -بسياري -جلوگيري -حق -كردند -نوعي -بعري -نكرده -نظير -نبايد -بوده -بودن -داد -اورد -هست -جايي -شود -دنبال -داده -بايد -سابق -هيچ -همان -انجا -كمتر -كجاست -گردد -كسي -تر -مردم -تان -دادن -بودند -سري -جدا -ندارند -مگر -يكديگر -دارد -دهند -بنابراين -هنگامي -سمت -جا -انچه -خود -دادند -زياد -دارند -اثر -بدون -بهترين -بيشتر -البته -به -براساس -بيرون -كرد -بعضي -گرفت -توي -اي -ميليون -او -جريان -تول -بر -مانند -برابر -باشيم -مدتي -گويند -اكنون -تا -تنها -جديد -چند -بي -نشده -كردن -كردم -گويد -كرده -كنيم -نمي -نزد -روي -قصد -فقط -بالاي -ديگران -اين -ديروز -توسط -سوم -ايم -دانند -سوي -استفاده -شما -كنار -داريم -ساخته -طور -امده -رفته -نخست -بيست -نزديك -طي -كنيد -از -انها -تمامي -داشت -يكي -طريق -اش -چيست -روب -نمايد -گفت -چندين -چيزي -تواند -ام -ايا -با -ان -ايد -ترين -اينكه -ديگري -راه -هايي -بروز -همچنان -پاعين -كس -حدود -مختلف -مقابل -چيز -گيرد -ندارد -ضد -همچون -سازي -شان -مورد -باره -مرسي -خويش -برخوردار -چون -خارج -شش -هنوز -تحت -ضمن -هستيم -گفته -فكر -بسيار -پيش -براي -روزهاي -انكه -نخواهد -بالا -كل -وقتي -كي -چنين -كه -گيري -نيست -است -كجا -كند -نيز -يابد -بندي -حتي -توانند -عقب -خواست -كنند -بين -تمام -همه -ما -باشند -مثل -شد -اري -باشد -اره -طبق -بعد -اگر -صورت -غير -جاي -بيش -ريزي -اند -زيرا -چگونه -بار -لطفا -مي -درباره -من -ديده -همين -گذاري -برداري -علت -گذاشته -هم -فوق -نه -ها -شوند -اباد -همواره -هر -اول -خواهند -چهار -نام -امروز -مان -هاي -قبل -كنم -سعي -تازه -را -هستند -زير -جلوي -عنوان -بود diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fi.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fi.txt deleted file mode 100644 index 4372c9a05..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fi.txt +++ /dev/null @@ -1,97 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - -| forms of BE - -olla -olen -olet -on -olemme -olette -ovat -ole | negative form - -oli -olisi -olisit -olisin -olisimme -olisitte -olisivat -olit -olin -olimme -olitte -olivat -ollut -olleet - -en | negation -et -ei -emme -ette -eivät - -|Nom Gen Acc Part Iness Elat Illat Adess Ablat Allat Ess Trans -minä minun minut minua minussa minusta minuun minulla minulta minulle | I -sinä sinun sinut sinua sinussa sinusta sinuun sinulla sinulta sinulle | you -hän hänen hänet häntä hänessä hänestä häneen hänellä häneltä hänelle | he she -me meidän meidät meitä meissä meistä meihin meillä meiltä meille | we -te teidän teidät teitä teissä teistä teihin teillä teiltä teille | you -he heidän heidät heitä heissä heistä heihin heillä heiltä heille | they - -tämä tämän tätä tässä tästä tähän tallä tältä tälle tänä täksi | this -tuo tuon tuotä tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that -se sen sitä siinä siitä siihen sillä siltä sille sinä siksi | it -nämä näiden näitä näissä näistä näihin näillä näiltä näille näinä näiksi | these -nuo noiden noita noissa noista noihin noilla noilta noille noina noiksi | those -ne niiden niitä niissä niistä niihin niillä niiltä niille niinä niiksi | they - -kuka kenen kenet ketä kenessä kenestä keneen kenellä keneltä kenelle kenenä keneksi| who -ketkä keiden ketkä keitä keissä keistä keihin keillä keiltä keille keinä keiksi | (pl) -mikä minkä minkä mitä missä mistä mihin millä miltä mille minä miksi | which what -mitkä | (pl) - -joka jonka jota jossa josta johon jolla jolta jolle jona joksi | who which -jotka joiden joita joissa joista joihin joilla joilta joille joina joiksi | (pl) - -| conjunctions - -että | that -ja | and -jos | if -koska | because -kuin | than -mutta | but -niin | so -sekä | and -sillä | for -tai | or -vaan | but -vai | or -vaikka | although - - -| prepositions - -kanssa | with -mukaan | according to -noin | about -poikki | across -yli | over, across - -| other - -kun | when -niin | so -nyt | now -itse | self - diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fr.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fr.txt deleted file mode 100644 index 749abae68..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_fr.txt +++ /dev/null @@ -1,186 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A French stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - -au | a + le -aux | a + les -avec | with -ce | this -ces | these -dans | with -de | of -des | de + les -du | de + le -elle | she -en | `of them' etc -et | and -eux | them -il | he -je | I -la | the -le | the -leur | their -lui | him -ma | my (fem) -mais | but -me | me -même | same; as in moi-même (myself) etc -mes | me (pl) -moi | me -mon | my (masc) -ne | not -nos | our (pl) -notre | our -nous | we -on | one -ou | where -par | by -pas | not -pour | for -qu | que before vowel -que | that -qui | who -sa | his, her (fem) -se | oneself -ses | his (pl) -son | his, her (masc) -sur | on -ta | thy (fem) -te | thee -tes | thy (pl) -toi | thee -ton | thy (masc) -tu | thou -un | a -une | a -vos | your (pl) -votre | your -vous | you - - | single letter forms - -c | c' -d | d' -j | j' -l | l' -à | to, at -m | m' -n | n' -s | s' -t | t' -y | there - - | forms of être (not including the infinitive): -été -étée -étées -étés -étant -suis -es -est -sommes -êtes -sont -serai -seras -sera -serons -serez -seront -serais -serait -serions -seriez -seraient -étais -était -étions -étiez -étaient -fus -fut -fûmes -fûtes -furent -sois -soit -soyons -soyez -soient -fusse -fusses -fût -fussions -fussiez -fussent - - | forms of avoir (not including the infinitive): -ayant -eu -eue -eues -eus -ai -as -avons -avez -ont -aurai -auras -aura -aurons -aurez -auront -aurais -aurait -aurions -auriez -auraient -avais -avait -avions -aviez -avaient -eut -eûmes -eûtes -eurent -aie -aies -ait -ayons -ayez -aient -eusse -eusses -eût -eussions -eussiez -eussent - - | Later additions (from Jean-Christophe Deschamps) -ceci | this -cela | that -celà | that -cet | this -cette | this -ici | here -ils | they -les | the (pl) -leurs | their (pl) -quel | which -quels | which -quelle | which -quelles | which -sans | without -soi | oneself - diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ga.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ga.txt deleted file mode 100644 index 9ff88d747..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ga.txt +++ /dev/null @@ -1,110 +0,0 @@ - -a -ach -ag -agus -an -aon -ar -arna -as -b' -ba -beirt -bhúr -caoga -ceathair -ceathrar -chomh -chtó -chuig -chun -cois -céad -cúig -cúigear -d' -daichead -dar -de -deich -deichniúr -den -dhá -do -don -dtí -dá -dár -dó -faoi -faoin -faoina -faoinár -fara -fiche -gach -gan -go -gur -haon -hocht -i -iad -idir -in -ina -ins -inár -is -le -leis -lena -lenár -m' -mar -mo -mé -na -nach -naoi -naonúr -ná -ní -níor -nó -nócha -ocht -ochtar -os -roimh -sa -seacht -seachtar -seachtó -seasca -seisear -siad -sibh -sinn -sna -sé -sí -tar -thar -thú -triúr -trí -trína -trínár -tríocha -tú -um -ár -é -éis -í -ó -ón -óna -ónár diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_gl.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_gl.txt deleted file mode 100644 index d8760b12c..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_gl.txt +++ /dev/null @@ -1,161 +0,0 @@ -# galican stopwords -a -aínda -alí -aquel -aquela -aquelas -aqueles -aquilo -aquí -ao -aos -as -así -á -ben -cando -che -co -coa -comigo -con -connosco -contigo -convosco -coas -cos -cun -cuns -cunha -cunhas -da -dalgunha -dalgunhas -dalgún -dalgúns -das -de -del -dela -delas -deles -desde -deste -do -dos -dun -duns -dunha -dunhas -e -el -ela -elas -eles -en -era -eran -esa -esas -ese -eses -esta -estar -estaba -está -están -este -estes -estiven -estou -eu -é -facer -foi -foron -fun -había -hai -iso -isto -la -las -lle -lles -lo -los -mais -me -meu -meus -min -miña -miñas -moi -na -nas -neste -nin -no -non -nos -nosa -nosas -noso -nosos -nós -nun -nunha -nuns -nunhas -o -os -ou -ó -ós -para -pero -pode -pois -pola -polas -polo -polos -por -que -se -senón -ser -seu -seus -sexa -sido -sobre -súa -súas -tamén -tan -te -ten -teñen -teño -ter -teu -teus -ti -tido -tiña -tiven -túa -túas -un -unha -unhas -uns -vos -vosa -vosas -voso -vosos -vós diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hi.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hi.txt deleted file mode 100644 index 86286bb08..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hi.txt +++ /dev/null @@ -1,235 +0,0 @@ -# Also see http://www.opensource.org/licenses/bsd-license.html -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# This file was created by Jacques Savoy and is distributed under the BSD license. -# Note: by default this file also contains forms normalized by HindiNormalizer -# for spelling variation (see section below), such that it can be used whether or -# not you enable that feature. When adding additional entries to this list, -# please add the normalized form as well. -अंदर -अत -अपना -अपनी -अपने -अभी -आदि -आप -इत्यादि -इन -इनका -इन्हीं -इन्हें -इन्हों -इस -इसका -इसकी -इसके -इसमें -इसी -इसे -उन -उनका -उनकी -उनके -उनको -उन्हीं -उन्हें -उन्हों -उस -उसके -उसी -उसे -एक -एवं -एस -ऐसे -और -कई -कर -करता -करते -करना -करने -करें -कहते -कहा -का -काफ़ी -कि -कितना -किन्हें -किन्हों -किया -किर -किस -किसी -किसे -की -कुछ -कुल -के -को -कोई -कौन -कौनसा -गया -घर -जब -जहाँ -जा -जितना -जिन -जिन्हें -जिन्हों -जिस -जिसे -जीधर -जैसा -जैसे -जो -तक -तब -तरह -तिन -तिन्हें -तिन्हों -तिस -तिसे -तो -था -थी -थे -दबारा -दिया -दुसरा -दूसरे -दो -द्वारा -न -नहीं -ना -निहायत -नीचे -ने -पर -पर -पहले -पूरा -पे -फिर -बनी -बही -बहुत -बाद -बाला -बिलकुल -भी -भीतर -मगर -मानो -मे -में -यदि -यह -यहाँ -यही -या -यिह -ये -रखें -रहा -रहे -ऱ्वासा -लिए -लिये -लेकिन -व -वर्ग -वह -वह -वहाँ -वहीं -वाले -वुह -वे -वग़ैरह -संग -सकता -सकते -सबसे -सभी -साथ -साबुत -साभ -सारा -से -सो -ही -हुआ -हुई -हुए -है -हैं -हो -होता -होती -होते -होना -होने -# additional normalized forms of the above -अपनि -जेसे -होति -सभि -तिंहों -इंहों -दवारा -इसि -किंहें -थि -उंहों -ओर -जिंहें -वहिं -अभि -बनि -हि -उंहिं -उंहें -हें -वगेरह -एसे -रवासा -कोन -निचे -काफि -उसि -पुरा -भितर -हे -बहि -वहां -कोइ -यहां -जिंहों -तिंहें -किसि -कइ -यहि -इंहिं -जिधर -इंहें -अदि -इतयादि -हुइ -कोनसा -इसकि -दुसरे -जहां -अप -किंहों -उनकि -भि -वरग -हुअ -जेसा -नहिं diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hu.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hu.txt deleted file mode 100644 index 37526da8a..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hu.txt +++ /dev/null @@ -1,211 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/hungarian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - -| Hungarian stop word list -| prepared by Anna Tordai - -a -ahogy -ahol -aki -akik -akkor -alatt -által -általában -amely -amelyek -amelyekben -amelyeket -amelyet -amelynek -ami -amit -amolyan -amíg -amikor -át -abban -ahhoz -annak -arra -arról -az -azok -azon -azt -azzal -azért -aztán -azután -azonban -bár -be -belül -benne -cikk -cikkek -cikkeket -csak -de -e -eddig -egész -egy -egyes -egyetlen -egyéb -egyik -egyre -ekkor -el -elég -ellen -elő -először -előtt -első -én -éppen -ebben -ehhez -emilyen -ennek -erre -ez -ezt -ezek -ezen -ezzel -ezért -és -fel -felé -hanem -hiszen -hogy -hogyan -igen -így -illetve -ill. -ill -ilyen -ilyenkor -ison -ismét -itt -jó -jól -jobban -kell -kellett -keresztül -keressünk -ki -kívül -között -közül -legalább -lehet -lehetett -legyen -lenne -lenni -lesz -lett -maga -magát -majd -majd -már -más -másik -meg -még -mellett -mert -mely -melyek -mi -mit -míg -miért -milyen -mikor -minden -mindent -mindenki -mindig -mint -mintha -mivel -most -nagy -nagyobb -nagyon -ne -néha -nekem -neki -nem -néhány -nélkül -nincs -olyan -ott -össze -ő -ők -őket -pedig -persze -rá -s -saját -sem -semmi -sok -sokat -sokkal -számára -szemben -szerint -szinte -talán -tehát -teljes -tovább -továbbá -több -úgy -ugyanis -új -újabb -újra -után -utána -utolsó -vagy -vagyis -valaki -valami -valamint -való -vagyok -van -vannak -volt -voltam -voltak -voltunk -vissza -vele -viszont -volna diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hy.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hy.txt deleted file mode 100644 index 60c1c50fb..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_hy.txt +++ /dev/null @@ -1,46 +0,0 @@ -# example set of Armenian stopwords. -այդ -այլ -այն -այս -դու -դուք -եմ -են -ենք -ես -եք -է -էի -էին -էինք -էիր -էիք -էր -ըստ -թ -ի -ին -իսկ -իր -կամ -համար -հետ -հետո -մենք -մեջ -մի -ն -նա -նաև -նրա -նրանք -որ -որը -որոնք -որպես -ու -ում -պիտի -վրա -և diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_id.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_id.txt deleted file mode 100644 index 4617f83a5..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_id.txt +++ /dev/null @@ -1,359 +0,0 @@ -# from appendix D of: A Study of Stemming Effects on Information -# Retrieval in Bahasa Indonesia -ada -adanya -adalah -adapun -agak -agaknya -agar -akan -akankah -akhirnya -aku -akulah -amat -amatlah -anda -andalah -antar -diantaranya -antara -antaranya -diantara -apa -apaan -mengapa -apabila -apakah -apalagi -apatah -atau -ataukah -ataupun -bagai -bagaikan -sebagai -sebagainya -bagaimana -bagaimanapun -sebagaimana -bagaimanakah -bagi -bahkan -bahwa -bahwasanya -sebaliknya -banyak -sebanyak -beberapa -seberapa -begini -beginian -beginikah -beginilah -sebegini -begitu -begitukah -begitulah -begitupun -sebegitu -belum -belumlah -sebelum -sebelumnya -sebenarnya -berapa -berapakah -berapalah -berapapun -betulkah -sebetulnya -biasa -biasanya -bila -bilakah -bisa -bisakah -sebisanya -boleh -bolehkah -bolehlah -buat -bukan -bukankah -bukanlah -bukannya -cuma -percuma -dahulu -dalam -dan -dapat -dari -daripada -dekat -demi -demikian -demikianlah -sedemikian -dengan -depan -di -dia -dialah -dini -diri -dirinya -terdiri -dong -dulu -enggak -enggaknya -entah -entahlah -terhadap -terhadapnya -hal -hampir -hanya -hanyalah -harus -haruslah -harusnya -seharusnya -hendak -hendaklah -hendaknya -hingga -sehingga -ia -ialah -ibarat -ingin -inginkah -inginkan -ini -inikah -inilah -itu -itukah -itulah -jangan -jangankan -janganlah -jika -jikalau -juga -justru -kala -kalau -kalaulah -kalaupun -kalian -kami -kamilah -kamu -kamulah -kan -kapan -kapankah -kapanpun -dikarenakan -karena -karenanya -ke -kecil -kemudian -kenapa -kepada -kepadanya -ketika -seketika -khususnya -kini -kinilah -kiranya -sekiranya -kita -kitalah -kok -lagi -lagian -selagi -lah -lain -lainnya -melainkan -selaku -lalu -melalui -terlalu -lama -lamanya -selama -selama -selamanya -lebih -terlebih -bermacam -macam -semacam -maka -makanya -makin -malah -malahan -mampu -mampukah -mana -manakala -manalagi -masih -masihkah -semasih -masing -mau -maupun -semaunya -memang -mereka -merekalah -meski -meskipun -semula -mungkin -mungkinkah -nah -namun -nanti -nantinya -nyaris -oleh -olehnya -seorang -seseorang -pada -padanya -padahal -paling -sepanjang -pantas -sepantasnya -sepantasnyalah -para -pasti -pastilah -per -pernah -pula -pun -merupakan -rupanya -serupa -saat -saatnya -sesaat -saja -sajalah -saling -bersama -sama -sesama -sambil -sampai -sana -sangat -sangatlah -saya -sayalah -se -sebab -sebabnya -sebuah -tersebut -tersebutlah -sedang -sedangkan -sedikit -sedikitnya -segala -segalanya -segera -sesegera -sejak -sejenak -sekali -sekalian -sekalipun -sesekali -sekaligus -sekarang -sekarang -sekitar -sekitarnya -sela -selain -selalu -seluruh -seluruhnya -semakin -sementara -sempat -semua -semuanya -sendiri -sendirinya -seolah -seperti -sepertinya -sering -seringnya -serta -siapa -siapakah -siapapun -disini -disinilah -sini -sinilah -sesuatu -sesuatunya -suatu -sesudah -sesudahnya -sudah -sudahkah -sudahlah -supaya -tadi -tadinya -tak -tanpa -setelah -telah -tentang -tentu -tentulah -tentunya -tertentu -seterusnya -tapi -tetapi -setiap -tiap -setidaknya -tidak -tidakkah -tidaklah -toh -waduh -wah -wahai -sewaktu -walau -walaupun -wong -yaitu -yakni -yang diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_it.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_it.txt deleted file mode 100644 index 1219cc773..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_it.txt +++ /dev/null @@ -1,303 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | An Italian stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - -ad | a (to) before vowel -al | a + il -allo | a + lo -ai | a + i -agli | a + gli -all | a + l' -agl | a + gl' -alla | a + la -alle | a + le -con | with -col | con + il -coi | con + i (forms collo, cogli etc are now very rare) -da | from -dal | da + il -dallo | da + lo -dai | da + i -dagli | da + gli -dall | da + l' -dagl | da + gll' -dalla | da + la -dalle | da + le -di | of -del | di + il -dello | di + lo -dei | di + i -degli | di + gli -dell | di + l' -degl | di + gl' -della | di + la -delle | di + le -in | in -nel | in + el -nello | in + lo -nei | in + i -negli | in + gli -nell | in + l' -negl | in + gl' -nella | in + la -nelle | in + le -su | on -sul | su + il -sullo | su + lo -sui | su + i -sugli | su + gli -sull | su + l' -sugl | su + gl' -sulla | su + la -sulle | su + le -per | through, by -tra | among -contro | against -io | I -tu | thou -lui | he -lei | she -noi | we -voi | you -loro | they -mio | my -mia | -miei | -mie | -tuo | -tua | -tuoi | thy -tue | -suo | -sua | -suoi | his, her -sue | -nostro | our -nostra | -nostri | -nostre | -vostro | your -vostra | -vostri | -vostre | -mi | me -ti | thee -ci | us, there -vi | you, there -lo | him, the -la | her, the -li | them -le | them, the -gli | to him, the -ne | from there etc -il | the -un | a -uno | a -una | a -ma | but -ed | and -se | if -perché | why, because -anche | also -come | how -dov | where (as dov') -dove | where -che | who, that -chi | who -cui | whom -non | not -più | more -quale | who, that -quanto | how much -quanti | -quanta | -quante | -quello | that -quelli | -quella | -quelle | -questo | this -questi | -questa | -queste | -si | yes -tutto | all -tutti | all - - | single letter forms: - -a | at -c | as c' for ce or ci -e | and -i | the -l | as l' -o | or - - | forms of avere, to have (not including the infinitive): - -ho -hai -ha -abbiamo -avete -hanno -abbia -abbiate -abbiano -avrò -avrai -avrà -avremo -avrete -avranno -avrei -avresti -avrebbe -avremmo -avreste -avrebbero -avevo -avevi -aveva -avevamo -avevate -avevano -ebbi -avesti -ebbe -avemmo -aveste -ebbero -avessi -avesse -avessimo -avessero -avendo -avuto -avuta -avuti -avute - - | forms of essere, to be (not including the infinitive): -sono -sei -è -siamo -siete -sia -siate -siano -sarò -sarai -sarà -saremo -sarete -saranno -sarei -saresti -sarebbe -saremmo -sareste -sarebbero -ero -eri -era -eravamo -eravate -erano -fui -fosti -fu -fummo -foste -furono -fossi -fosse -fossimo -fossero -essendo - - | forms of fare, to do (not including the infinitive, fa, fat-): -faccio -fai -facciamo -fanno -faccia -facciate -facciano -farò -farai -farà -faremo -farete -faranno -farei -faresti -farebbe -faremmo -fareste -farebbero -facevo -facevi -faceva -facevamo -facevate -facevano -feci -facesti -fece -facemmo -faceste -fecero -facessi -facesse -facessimo -facessero -facendo - - | forms of stare, to be (not including the infinitive): -sto -stai -sta -stiamo -stanno -stia -stiate -stiano -starò -starai -starà -staremo -starete -staranno -starei -staresti -starebbe -staremmo -stareste -starebbero -stavo -stavi -stava -stavamo -stavate -stavano -stetti -stesti -stette -stemmo -steste -stettero -stessi -stesse -stessimo -stessero -stando diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ja.txt deleted file mode 100644 index d4321be6b..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ja.txt +++ /dev/null @@ -1,127 +0,0 @@ -# -# This file defines a stopword set for Japanese. -# -# This set is made up of hand-picked frequent terms from segmented Japanese Wikipedia. -# Punctuation characters and frequent kanji have mostly been left out. See LUCENE-3745 -# for frequency lists, etc. that can be useful for making your own set (if desired) -# -# Note that there is an overlap between these stopwords and the terms stopped when used -# in combination with the JapanesePartOfSpeechStopFilter. When editing this file, note -# that comments are not allowed on the same line as stopwords. -# -# Also note that stopping is done in a case-insensitive manner. Change your StopFilter -# configuration if you need case-sensitive stopping. Lastly, note that stopping is done -# using the same character width as the entries in this file. Since this StopFilter is -# normally done after a CJKWidthFilter in your chain, you would usually want your romaji -# entries to be in half-width and your kana entries to be in full-width. -# -の -に -は -を -た -が -で -て -と -し -れ -さ -ある -いる -も -する -から -な -こと -として -い -や -れる -など -なっ -ない -この -ため -その -あっ -よう -また -もの -という -あり -まで -られ -なる -へ -か -だ -これ -によって -により -おり -より -による -ず -なり -られる -において -ば -なかっ -なく -しかし -について -せ -だっ -その後 -できる -それ -う -ので -なお -のみ -でき -き -つ -における -および -いう -さらに -でも -ら -たり -その他 -に関する -たち -ます -ん -なら -に対して -特に -せる -及び -これら -とき -では -にて -ほか -ながら -うち -そして -とともに -ただし -かつて -それぞれ -または -お -ほど -ものの -に対する -ほとんど -と共に -といった -です -とも -ところ -ここ -##### End of file diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_lv.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_lv.txt deleted file mode 100644 index e21a23c06..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_lv.txt +++ /dev/null @@ -1,172 +0,0 @@ -# Set of Latvian stopwords from A Stemming Algorithm for Latvian, Karlis Kreslins -# the original list of over 800 forms was refined: -# pronouns, adverbs, interjections were removed -# -# prepositions -aiz -ap -ar -apakš -ārpus -augšpus -bez -caur -dēļ -gar -iekš -iz -kopš -labad -lejpus -līdz -no -otrpus -pa -par -pār -pēc -pie -pirms -pret -priekš -starp -šaipus -uz -viņpus -virs -virspus -zem -apakšpus -# Conjunctions -un -bet -jo -ja -ka -lai -tomēr -tikko -turpretī -arī -kaut -gan -tādēļ -tā -ne -tikvien -vien -kā -ir -te -vai -kamēr -# Particles -ar -diezin -droši -diemžēl -nebūt -ik -it -taču -nu -pat -tiklab -iekšpus -nedz -tik -nevis -turpretim -jeb -iekam -iekām -iekāms -kolīdz -līdzko -tiklīdz -jebšu -tālab -tāpēc -nekā -itin -jā -jau -jel -nē -nezin -tad -tikai -vis -tak -iekams -vien -# modal verbs -būt -biju -biji -bija -bijām -bijāt -esmu -esi -esam -esat -būšu -būsi -būs -būsim -būsiet -tikt -tiku -tiki -tika -tikām -tikāt -tieku -tiec -tiek -tiekam -tiekat -tikšu -tiks -tiksim -tiksiet -tapt -tapi -tapāt -topat -tapšu -tapsi -taps -tapsim -tapsiet -kļūt -kļuvu -kļuvi -kļuva -kļuvām -kļuvāt -kļūstu -kļūsti -kļūst -kļūstam -kļūstat -kļūšu -kļūsi -kļūs -kļūsim -kļūsiet -# verbs -varēt -varēju -varējām -varēšu -varēsim -var -varēji -varējāt -varēsi -varēsiet -varat -varēja -varēs diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_nl.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_nl.txt deleted file mode 100644 index 47a2aeacf..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_nl.txt +++ /dev/null @@ -1,119 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Dutch stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This is a ranked list (commonest to rarest) of stopwords derived from - | a large sample of Dutch text. - - | Dutch stop words frequently exhibit homonym clashes. These are indicated - | clearly below. - -de | the -en | and -van | of, from -ik | I, the ego -te | (1) chez, at etc, (2) to, (3) too -dat | that, which -die | that, those, who, which -in | in, inside -een | a, an, one -hij | he -het | the, it -niet | not, nothing, naught -zijn | (1) to be, being, (2) his, one's, its -is | is -was | (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river -op | on, upon, at, in, up, used up -aan | on, upon, to (as dative) -met | with, by -als | like, such as, when -voor | (1) before, in front of, (2) furrow -had | had, past tense all persons sing. of 'hebben' (have) -er | there -maar | but, only -om | round, about, for etc -hem | him -dan | then -zou | should/would, past tense all persons sing. of 'zullen' -of | or, whether, if -wat | what, something, anything -mijn | possessive and noun 'mine' -men | people, 'one' -dit | this -zo | so, thus, in this way -door | through by -over | over, across -ze | she, her, they, them -zich | oneself -bij | (1) a bee, (2) by, near, at -ook | also, too -tot | till, until -je | you -mij | me -uit | out of, from -der | Old Dutch form of 'van der' still found in surnames -daar | (1) there, (2) because -haar | (1) her, their, them, (2) hair -naar | (1) unpleasant, unwell etc, (2) towards, (3) as -heb | present first person sing. of 'to have' -hoe | how, why -heeft | present third person sing. of 'to have' -hebben | 'to have' and various parts thereof -deze | this -u | you -want | (1) for, (2) mitten, (3) rigging -nog | yet, still -zal | 'shall', first and third person sing. of verb 'zullen' (will) -me | me -zij | she, they -nu | now -ge | 'thou', still used in Belgium and south Netherlands -geen | none -omdat | because -iets | something, somewhat -worden | to become, grow, get -toch | yet, still -al | all, every, each -waren | (1) 'were' (2) to wander, (3) wares, (3) -veel | much, many -meer | (1) more, (2) lake -doen | to do, to make -toen | then, when -moet | noun 'spot/mote' and present form of 'to must' -ben | (1) am, (2) 'are' in interrogative second person singular of 'to be' -zonder | without -kan | noun 'can' and present form of 'to be able' -hun | their, them -dus | so, consequently -alles | all, everything, anything -onder | under, beneath -ja | yes, of course -eens | once, one day -hier | here -wie | who -werd | imperfect third person sing. of 'become' -altijd | always -doch | yet, but etc -wordt | present third person sing. of 'become' -wezen | (1) to be, (2) 'been' as in 'been fishing', (3) orphans -kunnen | to be able -ons | us/our -zelf | self -tegen | against, towards, at -na | after, near -reeds | already -wil | (1) present tense of 'want', (2) 'will', noun, (3) fender -kon | could; past tense of 'to be able' -niets | nothing -uw | your -iemand | somebody -geweest | been; past participle of 'be' -andere | other diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_no.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_no.txt deleted file mode 100644 index a7a2c28ba..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_no.txt +++ /dev/null @@ -1,194 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/norwegian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Norwegian stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This stop word list is for the dominant bokmål dialect. Words unique - | to nynorsk are marked *. - - | Revised by Jan Bruusgaard , Jan 2005 - -og | and -i | in -jeg | I -det | it/this/that -at | to (w. inf.) -en | a/an -et | a/an -den | it/this/that -til | to -er | is/am/are -som | who/that -på | on -de | they / you(formal) -med | with -han | he -av | of -ikke | not -ikkje | not * -der | there -så | so -var | was/were -meg | me -seg | you -men | but -ett | one -har | have -om | about -vi | we -min | my -mitt | my -ha | have -hadde | had -hun | she -nå | now -over | over -da | when/as -ved | by/know -fra | from -du | you -ut | out -sin | your -dem | them -oss | us -opp | up -man | you/one -kan | can -hans | his -hvor | where -eller | or -hva | what -skal | shall/must -selv | self (reflective) -sjøl | self (reflective) -her | here -alle | all -vil | will -bli | become -ble | became -blei | became * -blitt | have become -kunne | could -inn | in -når | when -være | be -kom | come -noen | some -noe | some -ville | would -dere | you -som | who/which/that -deres | their/theirs -kun | only/just -ja | yes -etter | after -ned | down -skulle | should -denne | this -for | for/because -deg | you -si | hers/his -sine | hers/his -sitt | hers/his -mot | against -å | to -meget | much -hvorfor | why -dette | this -disse | these/those -uten | without -hvordan | how -ingen | none -din | your -ditt | your -blir | become -samme | same -hvilken | which -hvilke | which (plural) -sånn | such a -inni | inside/within -mellom | between -vår | our -hver | each -hvem | who -vors | us/ours -hvis | whose -både | both -bare | only/just -enn | than -fordi | as/because -før | before -mange | many -også | also -slik | just -vært | been -være | to be -båe | both * -begge | both -siden | since -dykk | your * -dykkar | yours * -dei | they * -deira | them * -deires | theirs * -deim | them * -di | your (fem.) * -då | as/when * -eg | I * -ein | a/an * -eit | a/an * -eitt | a/an * -elles | or * -honom | he * -hjå | at * -ho | she * -hoe | she * -henne | her -hennar | her/hers -hennes | hers -hoss | how * -hossen | how * -ikkje | not * -ingi | noone * -inkje | noone * -korleis | how * -korso | how * -kva | what/which * -kvar | where * -kvarhelst | where * -kven | who/whom * -kvi | why * -kvifor | why * -me | we * -medan | while * -mi | my * -mine | my * -mykje | much * -no | now * -nokon | some (masc./neut.) * -noka | some (fem.) * -nokor | some * -noko | some * -nokre | some * -si | his/hers * -sia | since * -sidan | since * -so | so * -somt | some * -somme | some * -um | about* -upp | up * -vere | be * -vore | was * -verte | become * -vort | become * -varte | became * -vart | became * - diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_pt.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_pt.txt deleted file mode 100644 index acfeb01af..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_pt.txt +++ /dev/null @@ -1,253 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Portuguese stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - - | The following is a ranked list (commonest to rarest) of stopwords - | deriving from a large sample of text. - - | Extra words have been added at the end. - -de | of, from -a | the; to, at; her -o | the; him -que | who, that -e | and -do | de + o -da | de + a -em | in -um | a -para | for - | é from SER -com | with -não | not, no -uma | a -os | the; them -no | em + o -se | himself etc -na | em + a -por | for -mais | more -as | the; them -dos | de + os -como | as, like -mas | but - | foi from SER -ao | a + o -ele | he -das | de + as - | tem from TER -à | a + a -seu | his -sua | her -ou | or - | ser from SER -quando | when -muito | much - | há from HAV -nos | em + os; us -já | already, now - | está from EST -eu | I -também | also -só | only, just -pelo | per + o -pela | per + a -até | up to -isso | that -ela | he -entre | between - | era from SER -depois | after -sem | without -mesmo | same -aos | a + os - | ter from TER -seus | his -quem | whom -nas | em + as -me | me -esse | that -eles | they - | estão from EST -você | you - | tinha from TER - | foram from SER -essa | that -num | em + um -nem | nor -suas | her -meu | my -às | a + as -minha | my - | têm from TER -numa | em + uma -pelos | per + os -elas | they - | havia from HAV - | seja from SER -qual | which - | será from SER -nós | we - | tenho from TER -lhe | to him, her -deles | of them -essas | those -esses | those -pelas | per + as -este | this - | fosse from SER -dele | of him - - | other words. There are many contractions such as naquele = em+aquele, - | mo = me+o, but they are rare. - | Indefinite article plural forms are also rare. - -tu | thou -te | thee -vocês | you (plural) -vos | you -lhes | to them -meus | my -minhas -teu | thy -tua -teus -tuas -nosso | our -nossa -nossos -nossas - -dela | of her -delas | of them - -esta | this -estes | these -estas | these -aquele | that -aquela | that -aqueles | those -aquelas | those -isto | this -aquilo | that - - | forms of estar, to be (not including the infinitive): -estou -está -estamos -estão -estive -esteve -estivemos -estiveram -estava -estávamos -estavam -estivera -estivéramos -esteja -estejamos -estejam -estivesse -estivéssemos -estivessem -estiver -estivermos -estiverem - - | forms of haver, to have (not including the infinitive): -hei -há -havemos -hão -houve -houvemos -houveram -houvera -houvéramos -haja -hajamos -hajam -houvesse -houvéssemos -houvessem -houver -houvermos -houverem -houverei -houverá -houveremos -houverão -houveria -houveríamos -houveriam - - | forms of ser, to be (not including the infinitive): -sou -somos -são -era -éramos -eram -fui -foi -fomos -foram -fora -fôramos -seja -sejamos -sejam -fosse -fôssemos -fossem -for -formos -forem -serei -será -seremos -serão -seria -seríamos -seriam - - | forms of ter, to have (not including the infinitive): -tenho -tem -temos -tém -tinha -tínhamos -tinham -tive -teve -tivemos -tiveram -tivera -tivéramos -tenha -tenhamos -tenham -tivesse -tivéssemos -tivessem -tiver -tivermos -tiverem -terei -terá -teremos -terão -teria -teríamos -teriam diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ro.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ro.txt deleted file mode 100644 index 4fdee90a5..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ro.txt +++ /dev/null @@ -1,233 +0,0 @@ -# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -acea -aceasta -această -aceea -acei -aceia -acel -acela -acele -acelea -acest -acesta -aceste -acestea -aceşti -aceştia -acolo -acum -ai -aia -aibă -aici -al -ăla -ale -alea -ălea -altceva -altcineva -am -ar -are -aş -aşadar -asemenea -asta -ăsta -astăzi -astea -ăstea -ăştia -asupra -aţi -au -avea -avem -aveţi -azi -bine -bucur -bună -ca -că -căci -când -care -cărei -căror -cărui -cât -câte -câţi -către -câtva -ce -cel -ceva -chiar -cînd -cine -cineva -cît -cîte -cîţi -cîtva -contra -cu -cum -cumva -curând -curînd -da -dă -dacă -dar -datorită -de -deci -deja -deoarece -departe -deşi -din -dinaintea -dintr -dintre -drept -după -ea -ei -el -ele -eram -este -eşti -eu -face -fără -fi -fie -fiecare -fii -fim -fiţi -iar -ieri -îi -îl -îmi -împotriva -în -înainte -înaintea -încât -încît -încotro -între -întrucât -întrucît -îţi -la -lângă -le -li -lîngă -lor -lui -mă -mâine -mea -mei -mele -mereu -meu -mi -mine -mult -multă -mulţi -ne -nicăieri -nici -nimeni -nişte -noastră -noastre -noi -noştri -nostru -nu -ori -oricând -oricare -oricât -orice -oricînd -oricine -oricît -oricum -oriunde -până -pe -pentru -peste -pînă -poate -pot -prea -prima -primul -prin -printr -sa -să -săi -sale -sau -său -se -şi -sînt -sîntem -sînteţi -spre -sub -sunt -suntem -sunteţi -ta -tăi -tale -tău -te -ţi -ţie -tine -toată -toate -tot -toţi -totuşi -tu -un -una -unde -undeva -unei -unele -uneori -unor -vă -vi -voastră -voastre -voi -voştri -vostru -vouă -vreo -vreun diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ru.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ru.txt deleted file mode 100644 index 55271400c..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_ru.txt +++ /dev/null @@ -1,243 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/russian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | a russian stop word list. comments begin with vertical bar. each stop - | word is at the start of a line. - - | this is a ranked list (commonest to rarest) of stopwords derived from - | a large text sample. - - | letter `ё' is translated to `е'. - -и | and -в | in/into -во | alternative form -не | not -что | what/that -он | he -на | on/onto -я | i -с | from -со | alternative form -как | how -а | milder form of `no' (but) -то | conjunction and form of `that' -все | all -она | she -так | so, thus -его | him -но | but -да | yes/and -ты | thou -к | towards, by -у | around, chez -же | intensifier particle -вы | you -за | beyond, behind -бы | conditional/subj. particle -по | up to, along -только | only -ее | her -мне | to me -было | it was -вот | here is/are, particle -от | away from -меня | me -еще | still, yet, more -нет | no, there isnt/arent -о | about -из | out of -ему | to him -теперь | now -когда | when -даже | even -ну | so, well -вдруг | suddenly -ли | interrogative particle -если | if -уже | already, but homonym of `narrower' -или | or -ни | neither -быть | to be -был | he was -него | prepositional form of его -до | up to -вас | you accusative -нибудь | indef. suffix preceded by hyphen -опять | again -уж | already, but homonym of `adder' -вам | to you -сказал | he said -ведь | particle `after all' -там | there -потом | then -себя | oneself -ничего | nothing -ей | to her -может | usually with `быть' as `maybe' -они | they -тут | here -где | where -есть | there is/are -надо | got to, must -ней | prepositional form of ей -для | for -мы | we -тебя | thee -их | them, their -чем | than -была | she was -сам | self -чтоб | in order to -без | without -будто | as if -человек | man, person, one -чего | genitive form of `what' -раз | once -тоже | also -себе | to oneself -под | beneath -жизнь | life -будет | will be -ж | short form of intensifer particle `же' -тогда | then -кто | who -этот | this -говорил | was saying -того | genitive form of `that' -потому | for that reason -этого | genitive form of `this' -какой | which -совсем | altogether -ним | prepositional form of `его', `они' -здесь | here -этом | prepositional form of `этот' -один | one -почти | almost -мой | my -тем | instrumental/dative plural of `тот', `то' -чтобы | full form of `in order that' -нее | her (acc.) -кажется | it seems -сейчас | now -были | they were -куда | where to -зачем | why -сказать | to say -всех | all (acc., gen. preposn. plural) -никогда | never -сегодня | today -можно | possible, one can -при | by -наконец | finally -два | two -об | alternative form of `о', about -другой | another -хоть | even -после | after -над | above -больше | more -тот | that one (masc.) -через | across, in -эти | these -нас | us -про | about -всего | in all, only, of all -них | prepositional form of `они' (they) -какая | which, feminine -много | lots -разве | interrogative particle -сказала | she said -три | three -эту | this, acc. fem. sing. -моя | my, feminine -впрочем | moreover, besides -хорошо | good -свою | ones own, acc. fem. sing. -этой | oblique form of `эта', fem. `this' -перед | in front of -иногда | sometimes -лучше | better -чуть | a little -том | preposn. form of `that one' -нельзя | one must not -такой | such a one -им | to them -более | more -всегда | always -конечно | of course -всю | acc. fem. sing of `all' -между | between - - - | b: some paradigms - | - | personal pronouns - | - | я меня мне мной [мною] - | ты тебя тебе тобой [тобою] - | он его ему им [него, нему, ним] - | она ее эи ею [нее, нэи, нею] - | оно его ему им [него, нему, ним] - | - | мы нас нам нами - | вы вас вам вами - | они их им ими [них, ним, ними] - | - | себя себе собой [собою] - | - | demonstrative pronouns: этот (this), тот (that) - | - | этот эта это эти - | этого эты это эти - | этого этой этого этих - | этому этой этому этим - | этим этой этим [этою] этими - | этом этой этом этих - | - | тот та то те - | того ту то те - | того той того тех - | тому той тому тем - | тем той тем [тою] теми - | том той том тех - | - | determinative pronouns - | - | (a) весь (all) - | - | весь вся все все - | всего всю все все - | всего всей всего всех - | всему всей всему всем - | всем всей всем [всею] всеми - | всем всей всем всех - | - | (b) сам (himself etc) - | - | сам сама само сами - | самого саму само самих - | самого самой самого самих - | самому самой самому самим - | самим самой самим [самою] самими - | самом самой самом самих - | - | stems of verbs `to be', `to have', `to do' and modal - | - | быть бы буд быв есть суть - | име - | дел - | мог мож мочь - | уме - | хоч хот - | долж - | можн - | нужн - | нельзя - diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_sv.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_sv.txt deleted file mode 100644 index 096f87f67..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_sv.txt +++ /dev/null @@ -1,133 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/swedish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Swedish stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This is a ranked list (commonest to rarest) of stopwords derived from - | a large text sample. - - | Swedish stop words occasionally exhibit homonym clashes. For example - | så = so, but also seed. These are indicated clearly below. - -och | and -det | it, this/that -att | to (with infinitive) -i | in, at -en | a -jag | I -hon | she -som | who, that -han | he -på | on -den | it, this/that -med | with -var | where, each -sig | him(self) etc -för | for -så | so (also: seed) -till | to -är | is -men | but -ett | a -om | if; around, about -hade | had -de | they, these/those -av | of -icke | not, no -mig | me -du | you -henne | her -då | then, when -sin | his -nu | now -har | have -inte | inte någon = no one -hans | his -honom | him -skulle | 'sake' -hennes | her -där | there -min | my -man | one (pronoun) -ej | nor -vid | at, by, on (also: vast) -kunde | could -något | some etc -från | from, off -ut | out -när | when -efter | after, behind -upp | up -vi | we -dem | them -vara | be -vad | what -över | over -än | than -dig | you -kan | can -sina | his -här | here -ha | have -mot | towards -alla | all -under | under (also: wonder) -någon | some etc -eller | or (else) -allt | all -mycket | much -sedan | since -ju | why -denna | this/that -själv | myself, yourself etc -detta | this/that -åt | to -utan | without -varit | was -hur | how -ingen | no -mitt | my -ni | you -bli | to be, become -blev | from bli -oss | us -din | thy -dessa | these/those -några | some etc -deras | their -blir | from bli -mina | my -samma | (the) same -vilken | who, that -er | you, your -sådan | such a -vår | our -blivit | from bli -dess | its -inom | within -mellan | between -sådant | such a -varför | why -varje | each -vilka | who, that -ditt | thy -vem | who -vilket | who, that -sitta | his -sådana | such a -vart | each -dina | thy -vars | whose -vårt | our -våra | our -ert | your -era | your -vilkas | whose - diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_th.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_th.txt deleted file mode 100644 index 07f0fabe6..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_th.txt +++ /dev/null @@ -1,119 +0,0 @@ -# Thai stopwords from: -# "Opinion Detection in Thai Political News Columns -# Based on Subjectivity Analysis" -# Khampol Sukhum, Supot Nitsuwat, and Choochart Haruechaiyasak -ไว้ -ไม่ -ไป -ได้ -ให้ -ใน -โดย -แห่ง -แล้ว -และ -แรก -แบบ -แต่ -เอง -เห็น -เลย -เริ่ม -เรา -เมื่อ -เพื่อ -เพราะ -เป็นการ -เป็น -เปิดเผย -เปิด -เนื่องจาก -เดียวกัน -เดียว -เช่น -เฉพาะ -เคย -เข้า -เขา -อีก -อาจ -อะไร -ออก -อย่าง -อยู่ -อยาก -หาก -หลาย -หลังจาก -หลัง -หรือ -หนึ่ง -ส่วน -ส่ง -สุด -สําหรับ -ว่า -วัน -ลง -ร่วม -ราย -รับ -ระหว่าง -รวม -ยัง -มี -มาก -มา -พร้อม -พบ -ผ่าน -ผล -บาง -น่า -นี้ -นํา -นั้น -นัก -นอกจาก -ทุก -ที่สุด -ที่ -ทําให้ -ทํา -ทาง -ทั้งนี้ -ทั้ง -ถ้า -ถูก -ถึง -ต้อง -ต่างๆ -ต่าง -ต่อ -ตาม -ตั้งแต่ -ตั้ง -ด้าน -ด้วย -ดัง -ซึ่ง -ช่วง -จึง -จาก -จัด -จะ -คือ -ความ -ครั้ง -คง -ขึ้น -ของ -ขอ -ขณะ -ก่อน -ก็ -การ -กับ -กัน -กว่า -กล่าว diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_tr.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_tr.txt deleted file mode 100644 index 84d9408d4..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/stopwords_tr.txt +++ /dev/null @@ -1,212 +0,0 @@ -# Turkish stopwords from LUCENE-559 -# merged with the list from "Information Retrieval on Turkish Texts" -# (http://www.users.muohio.edu/canf/papers/JASIST2008offPrint.pdf) -acaba -altmış -altı -ama -ancak -arada -aslında -ayrıca -bana -bazı -belki -ben -benden -beni -benim -beri -beş -bile -bin -bir -birçok -biri -birkaç -birkez -birşey -birşeyi -biz -bize -bizden -bizi -bizim -böyle -böylece -bu -buna -bunda -bundan -bunlar -bunları -bunların -bunu -bunun -burada -çok -çünkü -da -daha -dahi -de -defa -değil -diğer -diye -doksan -dokuz -dolayı -dolayısıyla -dört -edecek -eden -ederek -edilecek -ediliyor -edilmesi -ediyor -eğer -elli -en -etmesi -etti -ettiği -ettiğini -gibi -göre -halen -hangi -hatta -hem -henüz -hep -hepsi -her -herhangi -herkesin -hiç -hiçbir -için -iki -ile -ilgili -ise -işte -itibaren -itibariyle -kadar -karşın -katrilyon -kendi -kendilerine -kendini -kendisi -kendisine -kendisini -kez -ki -kim -kimden -kime -kimi -kimse -kırk -milyar -milyon -mu -mü -mı -nasıl -ne -neden -nedenle -nerde -nerede -nereye -niye -niçin -o -olan -olarak -oldu -olduğu -olduğunu -olduklarını -olmadı -olmadığı -olmak -olması -olmayan -olmaz -olsa -olsun -olup -olur -olursa -oluyor -on -ona -ondan -onlar -onlardan -onları -onların -onu -onun -otuz -oysa -öyle -pek -rağmen -sadece -sanki -sekiz -seksen -sen -senden -seni -senin -siz -sizden -sizi -sizin -şey -şeyden -şeyi -şeyler -şöyle -şu -şuna -şunda -şundan -şunları -şunu -tarafından -trilyon -tüm -üç -üzere -var -vardı -ve -veya -ya -yani -yapacak -yapılan -yapılması -yapıyor -yapmak -yaptı -yaptığı -yaptığını -yaptıkları -yedi -yerine -yetmiş -yine -yirmi -yoksa -yüz -zaten diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/userdict_ja.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/userdict_ja.txt deleted file mode 100644 index 6f0368e4d..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/lang/userdict_ja.txt +++ /dev/null @@ -1,29 +0,0 @@ -# -# This is a sample user dictionary for Kuromoji (JapaneseTokenizer) -# -# Add entries to this file in order to override the statistical model in terms -# of segmentation, readings and part-of-speech tags. Notice that entries do -# not have weights since they are always used when found. This is by-design -# in order to maximize ease-of-use. -# -# Entries are defined using the following CSV format: -# , ... , ... , -# -# Notice that a single half-width space separates tokens and readings, and -# that the number tokens and readings must match exactly. -# -# Also notice that multiple entries with the same is undefined. -# -# Whitespace only lines are ignored. Comments are not allowed on entry lines. -# - -# Custom segmentation for kanji compounds -日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞 -関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,カスタム名詞 - -# Custom segmentation for compound katakana -トートバッグ,トート バッグ,トート バッグ,かずカナ名詞 -ショルダーバッグ,ショルダー バッグ,ショルダー バッグ,かずカナ名詞 - -# Custom reading for former sumo wrestler -朝青龍,朝青龍,アサショウリュウ,カスタム人名 diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/protwords.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/protwords.txt deleted file mode 100644 index 1dfc0abec..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/protwords.txt +++ /dev/null @@ -1,21 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# Use a protected word file to protect against the stemmer reducing two -# unrelated words to the same base word. - -# Some non-words that normally won't be encountered, -# just to test that they won't be stemmed. -dontstems -zwhacky - diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/schema.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/schema.xml deleted file mode 100644 index 024c6ebb2..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/schema.xml +++ /dev/null @@ -1,766 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/spellings.txt b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/spellings.txt deleted file mode 100644 index d7ede6f56..000000000 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/spellings.txt +++ /dev/null @@ -1,2 +0,0 @@ -pizza -history \ No newline at end of file From 0b32e93ce57ebcaa5bbaa7b7cf6ba1f36ddbce08 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Fri, 8 Nov 2019 17:21:06 +0100 Subject: [PATCH 68/76] [ SEARCH-1917 ] Working implementation + first 10 Unit tests --- .../solr/AlfrescoCoreAdminHandler.java | 1330 +++++++++-------- ...tBuilder.java => HandlerReportHelper.java} | 10 +- .../java/org/alfresco/solr/utils/Utils.java | 56 + .../solr/AlfrescoCoreAdminHandlerTest.java | 88 +- 4 files changed, 813 insertions(+), 671 deletions(-) rename search-services/alfresco-search/src/main/java/org/alfresco/solr/{HandlerReportBuilder.java => HandlerReportHelper.java} (98%) create mode 100644 search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index b8e5c36b2..e43d9b536 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -19,60 +19,25 @@ package org.alfresco.solr; -import static java.util.Optional.ofNullable; - -import static org.alfresco.solr.HandlerOfResources.extractCustomProperties; -import static org.alfresco.solr.HandlerOfResources.getSafeBoolean; -import static org.alfresco.solr.HandlerOfResources.getSafeLong; -import static org.alfresco.solr.HandlerOfResources.openResource; -import static org.alfresco.solr.HandlerOfResources.updatePropertiesFile; -import static org.alfresco.solr.HandlerOfResources.updateSharedProperties; -import static org.alfresco.solr.HandlerReportBuilder.addMasterOrStandaloneCoreSummary; -import static org.alfresco.solr.HandlerReportBuilder.addSlaveCoreSummary; -import static org.alfresco.solr.HandlerReportBuilder.buildAclReport; -import static org.alfresco.solr.HandlerReportBuilder.buildAclTxReport; -import static org.alfresco.solr.HandlerReportBuilder.buildNodeReport; -import static org.alfresco.solr.HandlerReportBuilder.buildTrackerReport; -import static org.alfresco.solr.HandlerReportBuilder.buildTxReport; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - import com.google.common.collect.ImmutableMap; - import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.httpclient.AuthenticationException; import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.solr.adapters.IOpenBitSet; import org.alfresco.solr.client.SOLRAPIClientFactory; import org.alfresco.solr.config.ConfigUtil; import org.alfresco.solr.tracker.AclTracker; +import org.alfresco.solr.tracker.CoreStatePublisher; import org.alfresco.solr.tracker.DBIDRangeRouter; import org.alfresco.solr.tracker.DocRouter; import org.alfresco.solr.tracker.IndexHealthReport; import org.alfresco.solr.tracker.MetadataTracker; -import org.alfresco.solr.tracker.CoreStatePublisher; import org.alfresco.solr.tracker.SlaveCoreStatePublisher; import org.alfresco.solr.tracker.SolrTrackerScheduler; import org.alfresco.solr.tracker.Tracker; import org.alfresco.solr.tracker.TrackerRegistry; +import org.alfresco.solr.utils.Utils; +import org.alfresco.util.Pair; import org.alfresco.util.shard.ExplicitShardingPolicy; -import org.apache.commons.codec.EncoderException; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.io.FileUtils; import org.apache.solr.common.SolrException; @@ -89,14 +54,75 @@ import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.Arrays.asList; +import static java.util.Arrays.stream; +import static java.util.Optional.ofNullable; +import static org.alfresco.solr.HandlerOfResources.extractCustomProperties; +import static org.alfresco.solr.HandlerOfResources.getSafeBoolean; +import static org.alfresco.solr.HandlerOfResources.getSafeLong; +import static org.alfresco.solr.HandlerOfResources.openResource; +import static org.alfresco.solr.HandlerOfResources.updatePropertiesFile; +import static org.alfresco.solr.HandlerOfResources.updateSharedProperties; +import static org.alfresco.solr.HandlerReportHelper.addMasterOrStandaloneCoreSummary; +import static org.alfresco.solr.HandlerReportHelper.addSlaveCoreSummary; +import static org.alfresco.solr.HandlerReportHelper.buildAclReport; +import static org.alfresco.solr.HandlerReportHelper.buildAclTxReport; +import static org.alfresco.solr.HandlerReportHelper.buildNodeReport; +import static org.alfresco.solr.HandlerReportHelper.buildTrackerReport; +import static org.alfresco.solr.HandlerReportHelper.buildTxReport; +import static org.alfresco.solr.utils.Utils.notNullOrEmpty; + /** * Alfresco Solr administration endpoints provider. * A customisation of the existing Solr {@link CoreAdminHandler} which offers additional administration endpoints. + * + * Since 1.5 the behaviour of these endpoints differs a bit depending on the target core. This because a lot of these + * endpoints rely on the information obtained from the trackers, and trackers (see SEARCH-1606) are disabled on slave + * cores. + * + * When a request arrives to this handler, the following are the possible scenarios: + * + *
      + *
    • + * a core is specified in the request: if the target core is a slave then a minimal response or an empty + * response with an informational message is returned. If instead the core is a master (or it is a standalone + * core) the service will return as much information as possible (as it happened before 1.5) + *
      • + *
    • + * a core isn't specified in the request: the request is supposed to target all available cores. However, while + * looping, slave cores are filtered out. In case all cores are slave (i.e. we are running a "pure" slave node) + * the response will be empty, it will include an informational message in order to warn the requestor. + * Sometimes this informative behaviour is not feasible: in those cases an empty response will be returned. + *
      • + *
    + * + * @author Andrea Gazzarini */ public class AlfrescoCoreAdminHandler extends CoreAdminHandler { protected static final Logger LOGGER = LoggerFactory.getLogger(AlfrescoCoreAdminHandler.class); - + + private static final String REPORT = "report"; private static final String ARG_ACLTXID = "acltxid"; static final String ARG_TXID = "txid"; private static final String ARG_ACLID = "aclid"; @@ -118,8 +144,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler private SolrTrackerScheduler scheduler; private TrackerRegistry trackerRegistry; - private ConcurrentHashMap informationServers = null; - + private ConcurrentHashMap informationServers; + + private static List CORE_PARAMETER_NAMES = asList(CoreAdminParams.CORE, "coreName", "index"); + public AlfrescoCoreAdminHandler() { super(); @@ -203,34 +231,34 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler /** * Shut down services that exist outside of the core. */ - public void shutdown() + @Override + public void shutdown() { super.shutdown(); - try + try { LOGGER.info("Shutting down Alfresco core container services"); + AlfrescoSolrDataModel.getInstance().close(); SOLRAPIClientFactory.close(); MultiThreadedHttpConnectionManager.shutdownAll(); - //Remove any core trackers still hanging around - trackerRegistry.getCoreNames().forEach(coreName -> trackerRegistry.removeTrackersForCore(coreName)); - - //Remove any information servers + coreNames().forEach(trackerRegistry::removeTrackersForCore); informationServers.clear(); - //Shutdown the scheduler and model tracker. if (!scheduler.isShutdown()) { scheduler.pauseAll(); + if (trackerRegistry.getModelTracker() != null) trackerRegistry.getModelTracker().shutdown(); + trackerRegistry.setModelTracker(null); scheduler.shutdown(); } - } - catch(Exception e) + } + catch(Exception exception) { - LOGGER.error("Problem shutting down", e); + LOGGER.error("Problem shutting down Alfresco core container services", exception); } } @@ -258,17 +286,18 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler protected void handleCustomAction(SolrQueryRequest req, SolrQueryResponse rsp) { SolrParams params = req.getParams(); - String cname = params.get(CoreAdminParams.CORE); String action = params.get(CoreAdminParams.ACTION); - action = action==null?"":action.toUpperCase(); + action = Objects.requireNonNullElse(action.toUpperCase(), ""); try { switch (action) { case "NEWCORE": + case "NEWINDEX": newCore(req, rsp); break; case "UPDATECORE": + case "UPDATEINDEX": updateCore(req); break; case "UPDATESHARED": @@ -278,121 +307,55 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler removeCore(req); break; case "NEWDEFAULTINDEX": + case "NEWDEFAULTCORE": newDefaultCore(req, rsp); break; case "CHECK": - actionCHECK(cname); + actionCHECK(params); break; case "NODEREPORT": - actionNODEREPORTS(rsp, params, cname); + actionNODEREPORTS(rsp, params); break; case "ACLREPORT": - actionACLREPORT(rsp, params, cname); + actionACLREPORT(rsp, params); break; case "TXREPORT": - actionTXREPORT(rsp, params, cname); + actionTXREPORT(rsp, params); break; case "ACLTXREPORT": - actionACLTXREPORT(rsp, params, cname); + actionACLTXREPORT(rsp, params); break; case "RANGECHECK": - rangeCheck(rsp, cname); + rangeCheck(rsp, params); break; case "EXPAND": - expand(rsp, params, cname); + expand(rsp, params); break; case "REPORT": - actionREPORT(rsp, params, cname); + actionREPORT(rsp, params); break; case "PURGE": - if (cname != null) - { - actionPURGE(params, cname); - } - else - { - for (String coreName : getTrackerRegistry().getCoreNames()) - { - actionPURGE(params, coreName); - } - } + actionPURGE(params); break; case "REINDEX": - if (cname != null) - { - actionREINDEX(params, cname); - } - else - { - for (String coreName : getTrackerRegistry().getCoreNames()) - { - actionREINDEX(params, coreName); - } - } + actionREINDEX(params); break; case "RETRY": - if (cname != null) - { - actionRETRY(rsp, cname); - } - else - { - for (String coreName : getTrackerRegistry().getCoreNames()) - { - actionRETRY(rsp, coreName); - } - } + actionRETRY(rsp, params); break; case "INDEX": - if (cname != null) - { - actionINDEX(params, cname); - } - else - { - for (String coreName : getTrackerRegistry().getCoreNames()) - { - actionINDEX(params, coreName); - } - } + actionINDEX(params); break; case "FIX": - if (cname != null) - { - actionFIX(cname); - } - else - { - for (String coreName : getTrackerRegistry().getCoreNames()) - { - actionFIX(coreName); - } - } + actionFIX(params); break; case "SUMMARY": - if (cname != null) - { - NamedList report = new SimpleOrderedMap<>(); - actionSUMMARY(params, report, cname); - rsp.add("Summary", report); - } - else - { - NamedList report = new SimpleOrderedMap<>(); - for (String coreName : getTrackerRegistry().getCoreNames()) - { - actionSUMMARY(params, report, coreName); - } - rsp.add("Summary", report); - } + actionSUMMARY(rsp, params); break; case "LOG4J": - String resource = "log4j-solr.properties"; - if (params.get("resource") != null) - { - resource = params.get("resource"); - } - initResourceBasedLogging(resource); + initResourceBasedLogging( + ofNullable(params.get("resource")) + .orElse("log4j-solr.properties")); break; default: super.handleCustomAction(req, rsp); @@ -406,7 +369,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } - private boolean newCore(SolrQueryRequest req, SolrQueryResponse rsp) + private void newCore(SolrQueryRequest req, SolrQueryResponse rsp) { SolrParams params = req.getParams(); req.getContext(); @@ -417,7 +380,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler String store = params.get("storeRef"); if (store == null || store.trim().length() == 0) { - return false; + return; } StoreRef storeRef = new StoreRef(store); @@ -428,22 +391,24 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler int nodeInstance = params.getInt("nodeInstance", -1); int numNodes = params.getInt("numNodes", 1); - String coreName = params.get("coreName"); + String coreName = coreName(params); String shardIds = params.get("shardIds"); - Properties properties = extractCustomProperties(params); - return newCore(coreName, numShards, storeRef, templateName, replicationFactor, nodeInstance, numNodes, shardIds, properties, rsp); + newCore(coreName, numShards, storeRef, templateName, replicationFactor, nodeInstance, numNodes, shardIds, extractCustomProperties(params), rsp); } - private boolean newDefaultCore(SolrQueryRequest req, SolrQueryResponse response) + private void newDefaultCore(SolrQueryRequest req, SolrQueryResponse response) { SolrParams params = req.getParams(); - String coreName = params.get("coreName") != null?params.get("coreName"):"alfresco"; - String templateName = params.get("template") != null?params.get("template"): DEFAULT_TEMPLATE; + String coreName = ofNullable(coreName(params)).orElse(ALFRESCO_CORE_NAME); + String templateName = + params.get("template") != null + ? params.get("template") + : DEFAULT_TEMPLATE; Properties extraProperties = extractCustomProperties(params); - return newDefaultCore( + newDefaultCore( coreName, ofNullable(params.get("storeRef")) .map(StoreRef::new) @@ -453,12 +418,12 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler response); } - private boolean newDefaultCore(String coreName, StoreRef storeRef, String templateName, Properties extraProperties, SolrQueryResponse rsp) + private void newDefaultCore(String coreName, StoreRef storeRef, String templateName, Properties extraProperties, SolrQueryResponse rsp) { - return newCore(coreName, 1, storeRef, templateName, 1, 1, 1, null, extraProperties, rsp); + newCore(coreName, 1, storeRef, templateName, 1, 1, 1, null, extraProperties, rsp); } - protected boolean newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) + protected void newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) { try { @@ -476,14 +441,14 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler collectionName = templateName + "--" + coreName + "--shards--"+numShards + "-x-"+replicationFactor+"--node--"+nodeInstance+"-of-"+numNodes; coreBase = coreName + "-"; } - - File baseDirectory = new File(solrHome, collectionName); - + + File baseDirectory = new File(solrHome, collectionName); + if(nodeInstance == -1) { - return false; + return; } - + List shards; if(shardIds != null) { @@ -494,11 +459,11 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler ExplicitShardingPolicy policy = new ExplicitShardingPolicy(numShards, replicationFactor, numNodes); if(!policy.configurationIsValid()) { - return false; + return; } shards = policy.getShardIdsForNode(nodeInstance); } - + for(Integer shard : shards) { coreName = coreBase + shard; @@ -517,8 +482,6 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } createAndRegisterNewCore(rsp, extraProperties, storeRef, template, solrCoreName, newCore, numShards, shard, templateName); } - - return true; } else { @@ -528,37 +491,41 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } File newCore = new File(solrHome, coreName); createAndRegisterNewCore(rsp, extraProperties, storeRef, template, coreName, newCore, 0, 0, templateName); - - return true; } - } - catch (IOException e) + catch (IOException exception) { - e.printStackTrace(); - return false; + LOGGER.error("I/O Failure detected while creating the new core " + + "(name={}, numShard={}, storeRef={}, template={}, replication factor={}, node instance={}, num nodes={}, shard ids={})", + coreName, + numShards, + storeRef, + templateName, + replicationFactor, + nodeInstance, + numNodes, + shardIds, + exception); } } - private List extractShards(String shardIds, int numShards) + /** + * Extracts the list of shard identifiers from the given input string. + * the "excludeFromShardId" parameter is used to filter out those shards whose identifier is equal or greater than + * that parameter. + * + * @param shardIds the shards input string, where shards are separated by comma. + * @param excludeFromShardId filter out those shards whose identifier is equal or greater than this value. + * @return the list of shard identifiers. + */ + List extractShards(String shardIds, int excludeFromShardId) { - List shards = new ArrayList<>(); - for(String shardId : shardIds.split(",")) - { - try - { - int shard = Integer.parseInt(shardId); - if(shard < numShards) - { - shards.add(shard); - } - } - catch(NumberFormatException nfe) - { - // ignore - } - } - return shards; + return stream(Objects.requireNonNullElse(shardIds, "").split(",")) + .map(String::trim) + .map(Utils::toIntOrNull) + .filter(Objects::nonNull) + .filter(shard -> shard < excludeFromShardId) + .collect(Collectors.toList()); } private void createAndRegisterNewCore(SolrQueryResponse rsp, Properties extraProperties, StoreRef storeRef, File template, String coreName, File newCore, int shardCount, int shardInstance, String templateName) throws IOException @@ -616,16 +583,15 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler rsp.add("core", core.getName()); } - private boolean hasAlfrescoCore(Collection cores) + boolean hasAlfrescoCore(Collection cores) { - if (cores == null || cores.isEmpty()) return false; - for (SolrCore core:cores) - { - if (trackerRegistry.hasTrackersForCore(core.getName())) return true; - } - return false; + return notNullOrEmpty(cores).stream() + .map(SolrCore::getName) + .anyMatch(trackerRegistry::hasTrackersForCore); } + // ::::: + private void updateShared(SolrQueryRequest req) { SolrParams params = req.getParams(); @@ -645,33 +611,25 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler private void updateCore(SolrQueryRequest req) { - String coreName = null; - SolrParams params = req.getParams(); + ofNullable(coreName(req.getParams())) + .map(String::trim) + .filter(coreName -> !coreName.isEmpty()) + .ifPresent(coreName -> { + try (SolrCore core = coreContainer.getCore(coreName)) + { - if (params.get("coreName") != null) - { - coreName = params.get("coreName"); - } - - if ((coreName == null) || (coreName.length() == 0)) - { - return; - } + if (core == null) + { + return; + } - try (SolrCore core = coreContainer.getCore(coreName)) - { + String configLocaltion = core.getResourceLoader().getConfigDir(); + File config = new File(configLocaltion, "solrcore.properties"); + updatePropertiesFile(req.getParams(), config, null); - if (core == null) - { - return; - } - - String configLocaltion = core.getResourceLoader().getConfigDir(); - File config = new File(configLocaltion, "solrcore.properties"); - updatePropertiesFile(params, config, null); - - coreContainer.reload(coreName); - } + coreContainer.reload(coreName); + } + }); } private void removeCore(SolrQueryRequest req) @@ -686,19 +644,423 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler if ((store == null) || (store.length() == 0)) { return; } StoreRef storeRef = new StoreRef(store); - String coreName = storeRef.getProtocol() + "-" + storeRef.getIdentifier(); - if (params.get("coreName") != null) - { - coreName = params.get("coreName"); - } - // remove core + String coreName = ofNullable(coreName(req.getParams())).orElse(storeRef.getProtocol() + "-" + storeRef.getIdentifier()); coreContainer.unload(coreName, true, true, true); } - private void actionFIX(String coreName) throws AuthenticationException, IOException, JSONException, EncoderException + private void actionCHECK(SolrParams params) { + String cname = params.get(CoreAdminParams.CORE); + coreNames().stream() + .filter(coreName -> cname == null || coreName.equals(cname)) + .map(trackerRegistry::getTrackersForCore) + .flatMap(Collection::stream) + .map(Tracker::getTrackerState) + .forEach(state -> state.setCheck(true)); + } + + private void actionNODEREPORTS(SolrQueryResponse rsp, SolrParams params) throws JSONException + { + Long dbid = + ofNullable(params.get(ARG_NODEID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No dbid parameter set.")); + + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + + coreNames().stream() + .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(trackerRegistry::hasTrackersForCore) + .map(coreName -> new Pair<>(coreName, nodeStatePublisher(coreName))) + .filter(coreNameAndPublisher -> coreNameAndPublisher.getSecond() != null) + .forEach(coreNameAndPublisher -> + report.add( + coreNameAndPublisher.getFirst(), + buildNodeReport(coreNameAndPublisher.getSecond(), dbid))); + } + + private void actionACLREPORT(SolrQueryResponse rsp, SolrParams params) throws JSONException + { + Long aclid = + ofNullable(params.get(ARG_ACLID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_ACLID + " parameter set.")); + + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + + coreNames().stream() + .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .map(coreName -> new Pair<>(coreName, trackerRegistry.getTrackerForCore(coreName, AclTracker.class))) + .filter(coreNameAndAclTracker -> coreNameAndAclTracker.getSecond() != null) + .forEach(coreNameAndAclTracker -> + report.add( + coreNameAndAclTracker.getFirst(), + buildAclReport(coreNameAndAclTracker.getSecond(), aclid))); + + if (report.size() == 0) + { + addAlertMessage(report); + } + } + + private void actionTXREPORT(SolrQueryResponse rsp, SolrParams params) throws JSONException + { + String coreName = + ofNullable(params.get(CoreAdminParams.CORE)) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + params.get(CoreAdminParams.CORE + " parameter set."))); + + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + if (isMasterOrStandalone(coreName)) + { + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + Long txid = + ofNullable(params.get(ARG_TXID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_TXID + " parameter set.")); + + report.add(coreName, buildTxReport(trackerRegistry, informationServers.get(coreName), coreName, tracker, txid)); + } + else + { + addAlertMessage(report); + } + } + + private void actionACLTXREPORT(SolrQueryResponse rsp, SolrParams params) throws JSONException + { + Long acltxid = + ofNullable(params.get(ARG_ACLTXID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_ACLTXID + " parameter set.")); + + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + + coreNames().stream() + .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .map(coreName -> new Pair<>(coreName, trackerRegistry.getTrackerForCore(coreName, AclTracker.class))) + .filter(coreNameAndAclTracker -> coreNameAndAclTracker.getSecond() != null) + .forEach(coreNameAndAclTracker -> + report.add( + coreNameAndAclTracker.getFirst(), + buildAclTxReport( + trackerRegistry, + informationServers.get(coreNameAndAclTracker.getFirst()), + coreNameAndAclTracker.getFirst(), + coreNameAndAclTracker.getSecond(), + acltxid))); + + if (report.size() == 0) + { + addAlertMessage(report); + } + } + + private void rangeCheck(SolrQueryResponse rsp, SolrParams params) throws IOException + { + String coreName = + ofNullable(params.get(CoreAdminParams.CORE)) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + params.get(CoreAdminParams.CORE + " parameter set."))); + + if (isMasterOrStandalone(coreName)) + { + InformationServer informationServer = informationServers.get(coreName); + + DocRouter docRouter = getDocRouter(coreName); + + if(docRouter instanceof DBIDRangeRouter) + { + DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter) docRouter; + + if(!dbidRangeRouter.getInitialized()) + { + rsp.add("expand", 0); + rsp.add("exception", "DBIDRangeRouter not initialized yet."); + return; + } + + long startRange = dbidRangeRouter.getStartRange(); + long endRange = dbidRangeRouter.getEndRange(); + + long maxNodeId = informationServer.maxNodeId(); + long minNodeId = informationServer.minNodeId(); + long nodeCount = informationServer.nodeCount(); + + long bestGuess = -1; // -1 means expansion cannot be done. Either because expansion + // has already happened or we're above safe range + + long range = endRange - startRange; // We want this many nodes on the server + + long midpoint = startRange + ((long) (range * .5)); + + long safe = startRange + ((long) (range * .75)); + + long offset = maxNodeId-startRange; + + double density = 0; + + if(offset > 0) + { + density = ((double)nodeCount) / ((double)offset); // This is how dense we are so far. + } + + if (!dbidRangeRouter.getExpanded()) + { + if(maxNodeId <= safe) + { + if (maxNodeId >= midpoint) + { + if(density >= 1 || density == 0) + { + //This is fully dense shard or an empty shard. + // If it does happen, no expand is required. + bestGuess=0; + } + else + { + double multiplier = 1/density; + bestGuess = (long)(range*multiplier)-range; // This is how much to add + } + } + else + { + bestGuess = 0; // We're below the midpoint so it's to early to make a guess. + } + } + } + + rsp.add("start", startRange); + rsp.add("end", endRange); + rsp.add("nodeCount", nodeCount); + rsp.add("minDbid", minNodeId); + rsp.add("maxDbid", maxNodeId); + rsp.add("density", Math.abs(density)); + rsp.add("expand", bestGuess); + rsp.add("expanded", dbidRangeRouter.getExpanded()); + } + else + { + rsp.add("expand", -1); + rsp.add("exception", "ERROR: Wrong document router type:"+docRouter.getClass().getSimpleName()); + } + } + else + { + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + addAlertMessage(report); + } + } + + private synchronized void expand(SolrQueryResponse rsp, SolrParams params) throws IOException + { + String coreName = + ofNullable(params.get(CoreAdminParams.CORE)) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + params.get(CoreAdminParams.CORE + " parameter set."))); + + if (isMasterOrStandalone(coreName)) + { + InformationServer informationServer = informationServers.get(coreName); + DocRouter docRouter = getDocRouter(coreName); + + if(docRouter instanceof DBIDRangeRouter) + { + long expansion = Long.parseLong(params.get("add")); + DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter)docRouter; + + if(!dbidRangeRouter.getInitialized()) + { + rsp.add("expand", -1); + rsp.add("exception", "DBIDRangeRouter not initialized yet."); + return; + } + + if(dbidRangeRouter.getExpanded()) + { + rsp.add("expand", -1); + rsp.add("exception", "dbid range has already been expanded."); + return; + } + + long currentEndRange = dbidRangeRouter.getEndRange(); + long startRange = dbidRangeRouter.getStartRange(); + long maxNodeId = informationServer.maxNodeId(); + + long range = currentEndRange - startRange; + long safe = startRange + ((long) (range * .75)); + + if(maxNodeId > safe) + { + rsp.add("expand", -1); + rsp.add("exception", "Expansion cannot occur if max DBID in the index is more then 75% of range."); + return; + } + + long newEndRange = expansion+dbidRangeRouter.getEndRange(); + try + { + informationServer.capIndex(newEndRange); + informationServer.hardCommit(); + dbidRangeRouter.setEndRange(newEndRange); + dbidRangeRouter.setExpanded(true); + assert newEndRange == dbidRangeRouter.getEndRange(); + rsp.add("expand", dbidRangeRouter.getEndRange()); + } + catch(Throwable t) + { + rsp.add("expand", -1); + rsp.add("exception", t.getMessage()); + LOGGER.error("exception expanding", t); + } + } + else + { + rsp.add("expand", -1); + rsp.add("exception", "Wrong document router type:" + docRouter.getClass().getSimpleName()); + } + } + else + { + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + addAlertMessage(report); + } + } + + private void actionREPORT(SolrQueryResponse rsp, SolrParams params) throws JSONException + { + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + + Long fromTime = getSafeLong(params, "fromTime"); + Long toTime = getSafeLong(params, "toTime"); + Long fromTx = getSafeLong(params, "fromTx"); + Long toTx = getSafeLong(params, "toTx"); + Long fromAclTx = getSafeLong(params, "fromAclTx"); + Long toAclTx = getSafeLong(params, "toAclTx"); + + coreNames().stream() + .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(trackerRegistry::hasTrackersForCore) + .filter(this::isMasterOrStandalone) + .forEach(coreName -> + report.add( + coreName, + buildTrackerReport( + trackerRegistry, + informationServers.get(coreName), + coreName, + fromTx, + toTx, + fromAclTx, + toAclTx, + fromTime, + toTime))); + + if (report.size() == 0) + { + addAlertMessage(report); + } + } + + private void actionPURGE(SolrParams params) + { + Consumer purgeOnSpecificCore = coreName -> { + final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + + apply(params, ARG_TXID, metadataTracker::addTransactionToPurge) + .andThen(apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToPurge)) + .andThen(apply(params, ARG_NODEID, metadataTracker::addNodeToPurge)) + .andThen(apply(params, ARG_ACLID, aclTracker::addAclToPurge)); + }; + + coreNames().stream() + .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(this::isMasterOrStandalone) + .forEach(purgeOnSpecificCore); + } + + private void actionREINDEX(SolrParams params) + { + Consumer reindexOnSpecificCore = coreName -> { + final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + + apply(params, ARG_TXID, metadataTracker::addTransactionToReindex) + .andThen(apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToReindex)) + .andThen(apply(params, ARG_NODEID, metadataTracker::addNodeToReindex)) + .andThen(apply(params, ARG_ACLID, aclTracker::addAclToReindex)); + + ofNullable(params.get(ARG_QUERY)).ifPresent(metadataTracker::addQueryToReindex); + }; + + coreNames().stream() + .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(this::isMasterOrStandalone) + .forEach(reindexOnSpecificCore); + } + + private void actionRETRY(SolrQueryResponse rsp, SolrParams params) + { + final Consumer retryOnSpecificCore = coreName -> { + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + InformationServer srv = informationServers.get(coreName); + + try + { + for (Long nodeid : srv.getErrorDocIds()) + { + tracker.addNodeToReindex(nodeid); + } + rsp.add(coreName, srv.getErrorDocIds()); + } + catch (Exception exception) + { + LOGGER.error("I/O Exception while adding Node to reindex.", exception); + } + }; + + coreNames().stream() + .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(this::isMasterOrStandalone) + .forEach(retryOnSpecificCore); + } + + private void actionINDEX(SolrParams params) + { + Consumer indexOnSpecificCore = coreName -> { + final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + + apply(params, ARG_TXID, metadataTracker::addTransactionToIndex) + .andThen(apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToIndex)) + .andThen(apply(params, ARG_NODEID, metadataTracker::addNodeToIndex)) + .andThen(apply(params, ARG_ACLID, aclTracker::addAclToIndex)); + }; + + coreNames().stream() + .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(this::isMasterOrStandalone) + .forEach(indexOnSpecificCore); + } + + private void actionFIX(SolrParams params) throws JSONException + { + coreNames().stream() + .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(this::isMasterOrStandalone) + .forEach(this::fixOnSpecificCore); + } + + private void fixOnSpecificCore(String coreName) + { + try { // Gets Metadata health and fixes any problems MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); @@ -708,8 +1070,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler toReindex.or(indexHealthReport.getMissingTxFromIndex()); long current = -1; // Goes through problems in the index - while ((current = toReindex.nextSetBit(current + 1)) != -1) - { + while ((current = toReindex.nextSetBit(current + 1)) != -1) { metadataTracker.addTransactionToReindex(current); } @@ -721,345 +1082,55 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler toReindex.or(indexHealthReport.getMissingAclTxFromIndex()); current = -1; // Goes through the problems in the index - while ((current = toReindex.nextSetBit(current + 1)) != -1) - { + while ((current = toReindex.nextSetBit(current + 1)) != -1) { aclTracker.addAclChangeSetToReindex(current); } } + catch(Exception exception) + { + throw new AlfrescoRuntimeException("", exception); + } } - private void actionCHECK(String cname) - { - trackerRegistry.getCoreNames() - .stream() - .filter(coreName -> cname == null || coreName.equals(cname)) - .map(trackerRegistry::getTrackersForCore) - .flatMap(Collection::stream) - .map(Tracker::getTrackerState) - .forEach(state -> state.setCheck(true)); - } - - private void actionACLREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws JSONException + private void actionSUMMARY(SolrQueryResponse rsp, SolrParams params) { NamedList report = new SimpleOrderedMap<>(); - rsp.add("report", report); + rsp.add("Summary", report); - Long aclid = - ofNullable(params.get(ARG_ACLID)) - .map(Long::valueOf) - .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_ACLID + " parameter set.")); - - if (cname != null) - { - ofNullable(trackerRegistry.getTrackerForCore(cname, AclTracker.class)) - .ifPresent(tracker -> report.add(cname, buildAclReport(tracker, aclid))); - } - else - { - trackerRegistry.getCoreNames() - .forEach(coreName -> - ofNullable(trackerRegistry.getTrackerForCore(coreName, AclTracker.class)) - .ifPresent(tracker -> report.add(coreName, buildAclReport(tracker, aclid)))); - } - - if (report.size() == 0) - { - report.add("WARNING", "This response comes from a slave core. Please consider to ask the same request to its corresponding master core, in order to get more information about the requested Node"); - } + coreNames().stream() + .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(this::isMasterOrStandalone) + .forEach(coreName -> coreSummary(params, report, coreName)); } - private void actionTXREPORT(SolrQueryResponse rsp, SolrParams params, String cname) - throws AuthenticationException, IOException, JSONException, EncoderException - { - NamedList report = new SimpleOrderedMap<>(); - rsp.add("report", report); - - MetadataTracker tracker = trackerRegistry.getTrackerForCore(cname, MetadataTracker.class); - if (tracker != null) - { - Long txid = - ofNullable(params.get(ARG_TXID)) - .map(Long::valueOf) - .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_TXID + " parameter set.")); - - if (cname == null) - { - throw new AlfrescoRuntimeException("No cname parameter set"); - } - - report.add(cname, buildTxReport(getTrackerRegistry(), informationServers.get(cname), cname, tracker, txid)); - } - else - { - report.add("WARNING", "This response comes from a slave core. Please consider to ask the same request to its corresponding master core, in order to get more information about the requested Node"); - } - } - - private void actionACLTXREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws JSONException - { - if (params.get(ARG_ACLTXID) == null) - { - throw new AlfrescoRuntimeException("No acltxid parameter set"); - } - - NamedList report = new SimpleOrderedMap<>(); - rsp.add("report", report); - - Long acltxid = - ofNullable(params.get(ARG_ACLTXID)) - .map(Long::valueOf) - .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_ACLTXID + " parameter set.")); - - if (cname != null) - { - ofNullable(trackerRegistry.getTrackerForCore(cname, AclTracker.class)) - .ifPresent(tracker -> report.add(cname, buildAclTxReport(trackerRegistry, informationServers.get(cname), cname, tracker, acltxid))); - } - else - { - trackerRegistry.getCoreNames() - .forEach(coreName -> - ofNullable(trackerRegistry.getTrackerForCore(coreName, AclTracker.class)) - .ifPresent(tracker -> report.add(cname, buildAclTxReport(trackerRegistry, informationServers.get(cname), cname, tracker, acltxid)))); - } - - if (report.size() == 0) - { - report.add("WARNING", "This response comes from a slave core. Please consider to ask the same request to its corresponding master core, in order to get more information about the requested Node"); - } - } - - private void actionREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws JSONException - { - NamedList report = new SimpleOrderedMap<>(); - rsp.add("report", report); - - Long fromTime = getSafeLong(params, "fromTime"); - Long toTime = getSafeLong(params, "toTime"); - Long fromTx = getSafeLong(params, "fromTx"); - Long toTx = getSafeLong(params, "toTx"); - Long fromAclTx = getSafeLong(params, "fromAclTx"); - Long toAclTx = getSafeLong(params, "toAclTx"); - - if (cname != null) - { - if (trackerRegistry.hasTrackersForCore(cname) && isMasterOrStandalone(cname)) - { - report.add(cname, buildTrackerReport(trackerRegistry, informationServers.get(cname),cname, fromTx, toTx, fromAclTx, toAclTx, fromTime, toTime)); - } - } - else - { - trackerRegistry.getCoreNames().stream() - .filter(trackerRegistry::hasTrackersForCore) - .filter(this::isMasterOrStandalone) - .forEach(coreName -> report.add(coreName, buildTrackerReport(trackerRegistry, informationServers.get(coreName), coreName, fromTx, toTx, fromAclTx, toAclTx, fromTime, toTime))); - } - } - - private DocRouter getDocRouter(String cname) - { - Collection trackers = trackerRegistry.getTrackersForCore(cname); - MetadataTracker metadataTracker = null; - for(Tracker tracker : trackers) - { - if(tracker instanceof MetadataTracker) - { - metadataTracker = (MetadataTracker)tracker; - } - } - - return metadataTracker.getDocRouter(); - } - - - private void rangeCheck(SolrQueryResponse rsp,String cname) throws IOException - { - InformationServer informationServer = informationServers.get(cname); - - DocRouter docRouter = getDocRouter(cname); - - if(docRouter instanceof DBIDRangeRouter) - { - DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter) docRouter; - - if(!dbidRangeRouter.getInitialized()) - { - rsp.add("expand", 0); - rsp.add("exception", "DBIDRangeRouter not initialized yet."); - return; - } - - long startRange = dbidRangeRouter.getStartRange(); - long endRange = dbidRangeRouter.getEndRange(); - - long maxNodeId = informationServer.maxNodeId(); - long minNodeId = informationServer.minNodeId(); - long nodeCount = informationServer.nodeCount(); - - long bestGuess = -1; // -1 means expansion cannot be done. Either because expansion - // has already happened or we're above safe range - - long range = endRange - startRange; // We want this many nodes on the server - - long midpoint = startRange + ((long) (range * .5)); - - long safe = startRange + ((long) (range * .75)); - - long offset = maxNodeId-startRange; - - double density = 0; - - if(offset > 0) - { - density = ((double)nodeCount) / ((double)offset); // This is how dense we are so far. - } - - if (!dbidRangeRouter.getExpanded()) - { - if(maxNodeId <= safe) - { - if (maxNodeId >= midpoint) - { - if(density >= 1 || density == 0) - { - //This is fully dense shard or an empty shard. - // If it does happen, no expand is required. - bestGuess=0; - } - else - { - double multiplier = 1/density; - bestGuess = (long)(range*multiplier)-range; // This is how much to add - } - } - else - { - bestGuess = 0; // We're below the midpoint so it's to early to make a guess. - } - } - } - - rsp.add("start", startRange); - rsp.add("end", endRange); - rsp.add("nodeCount", nodeCount); - rsp.add("minDbid", minNodeId); - rsp.add("maxDbid", maxNodeId); - rsp.add("density", Math.abs(density)); - rsp.add("expand", bestGuess); - rsp.add("expanded", dbidRangeRouter.getExpanded()); - } - else - { - rsp.add("expand", -1); - rsp.add("exception", "ERROR: Wrong document router type:"+docRouter.getClass().getSimpleName()); - } - } - - private synchronized void expand(SolrQueryResponse rsp, SolrParams params, String cname) throws IOException - { - InformationServer informationServer = informationServers.get(cname); - DocRouter docRouter = getDocRouter(cname); - - if(docRouter instanceof DBIDRangeRouter) - { - long expansion = Long.parseLong(params.get("add")); - DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter)docRouter; - - if(!dbidRangeRouter.getInitialized()) - { - rsp.add("expand", -1); - rsp.add("exception", "DBIDRangeRouter not initialized yet."); - return; - } - - if(dbidRangeRouter.getExpanded()) - { - rsp.add("expand", -1); - rsp.add("exception", "dbid range has already been expanded."); - return; - } - - long currentEndRange = dbidRangeRouter.getEndRange(); - long startRange = dbidRangeRouter.getStartRange(); - long maxNodeId = informationServer.maxNodeId(); - - long range = currentEndRange - startRange; - long safe = startRange + ((long) (range * .75)); - - if(maxNodeId > safe) - { - rsp.add("expand", -1); - rsp.add("exception", "Expansion cannot occur if max DBID in the index is more then 75% of range."); - return; - } - - long newEndRange = expansion+dbidRangeRouter.getEndRange(); - try - { - informationServer.capIndex(newEndRange); - informationServer.hardCommit(); - dbidRangeRouter.setEndRange(newEndRange); - dbidRangeRouter.setExpanded(true); - assert newEndRange == dbidRangeRouter.getEndRange(); - rsp.add("expand", dbidRangeRouter.getEndRange()); - } - catch(Throwable t) - { - rsp.add("expand", -1); - rsp.add("exception", t.getMessage()); - LOGGER.error("exception expanding", t); - } - } - else - { - rsp.add("expand", -1); - rsp.add("exception", "Wrong document router type:" + docRouter.getClass().getSimpleName()); - } - } - - private void actionNODEREPORTS(SolrQueryResponse rsp, SolrParams params, String cname) throws JSONException - { - Long dbid = - ofNullable(params.get(ARG_NODEID)) - .map(Long::valueOf) - .orElseThrow(() -> new AlfrescoRuntimeException("No dbid parameter set.")); - - NamedList report = new SimpleOrderedMap<>(); - rsp.add("report", report); - - if (cname != null) - { - report.add(cname, buildNodeReport(nodeStatePublisher(cname), dbid)); - } - else - { - trackerRegistry.getCoreNames().forEach(coreName -> report.add(coreName, buildNodeReport(nodeStatePublisher(coreName), dbid))); - } - } - - private void actionSUMMARY(SolrParams params, NamedList report, String coreName) throws IOException + private void coreSummary(SolrParams params, NamedList report, String coreName) { boolean detail = getSafeBoolean(params, "detail"); boolean hist = getSafeBoolean(params, "hist"); boolean values = getSafeBoolean(params, "values"); boolean reset = getSafeBoolean(params, "reset"); - + InformationServer srv = informationServers.get(coreName); if (srv != null) { - if (isMasterOrStandalone(coreName)) + try { - addMasterOrStandaloneCoreSummary(trackerRegistry, coreName, detail, hist, values, srv, report); - - if (reset) + if (isMasterOrStandalone(coreName)) { - srv.getTrackerStats().reset(); + addMasterOrStandaloneCoreSummary(trackerRegistry, coreName, detail, hist, values, srv, report); + + if (reset) + { + srv.getTrackerStats().reset(); + } + } else + { + addSlaveCoreSummary(trackerRegistry, coreName, detail, hist, values, srv, report); } } - else + catch(Exception exception) { - addSlaveCoreSummary(trackerRegistry, coreName, detail, hist, values, srv, report); + throw new AlfrescoRuntimeException("", exception); } } else @@ -1068,129 +1139,15 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } - private void actionINDEX(SolrParams params, String coreName) + private DocRouter getDocRouter(String cname) { - if (isMasterOrStandalone(coreName)) - { - if (params.get(ARG_TXID) != null) - { - Long txid = Long.valueOf(params.get(ARG_TXID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addTransactionToIndex(txid); - } - - if (params.get(ARG_ACLTXID) != null) - { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclChangeSetToIndex(acltxid); - } - - if (params.get(ARG_NODEID) != null) - { - Long nodeid = Long.valueOf(params.get(ARG_NODEID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addNodeToIndex(nodeid); - } - - if (params.get(ARG_ACLID) != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclToIndex(aclid); - } - } - } - - private void actionRETRY(SolrQueryResponse rsp, String coreName) throws IOException - { - if (isMasterOrStandalone(coreName)) - { - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - InformationServer srv = informationServers.get(coreName); - - Set errorDocIds = srv.getErrorDocIds(); - for (Long nodeid : errorDocIds) - { - tracker.addNodeToReindex(nodeid); - } - rsp.add(coreName, errorDocIds); - } - } - - private void actionREINDEX(SolrParams params, String coreName) - { - if (isMasterOrStandalone(coreName)) - { - if (params.get(ARG_TXID) != null) - { - Long txid = Long.valueOf(params.get(ARG_TXID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addTransactionToReindex(txid); - } - - if (params.get(ARG_ACLTXID) != null) - { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclChangeSetToReindex(acltxid); - } - - if (params.get(ARG_NODEID) != null) - { - Long nodeid = Long.valueOf(params.get(ARG_NODEID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addNodeToReindex(nodeid); - } - - if (params.get(ARG_ACLID) != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclToReindex(aclid); - } - - if (params.get(ARG_QUERY) != null) - { - String query = params.get(ARG_QUERY); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addQueryToReindex(query); - } - } - } - - private void actionPURGE(SolrParams params, String coreName) - { - if (isMasterOrStandalone(coreName)) - { - if (params.get(ARG_TXID) != null) - { - Long txid = Long.valueOf(params.get(ARG_TXID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addTransactionToPurge(txid); - } - - if (params.get(ARG_ACLTXID) != null) - { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclChangeSetToPurge(acltxid); - } - - if (params.get(ARG_NODEID) != null) - { - Long nodeid = Long.valueOf(params.get(ARG_NODEID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addNodeToPurge(nodeid); - } - - if (params.get(ARG_ACLID) != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclToPurge(aclid); - } - } + return notNullOrEmpty(trackerRegistry.getTrackersForCore(cname)) + .stream() + .filter(tracker -> tracker instanceof MetadataTracker) + .findAny() + .map(MetadataTracker.class::cast) + .map(MetadataTracker::getDocRouter) + .orElse(null); } public ConcurrentHashMap getInformationServers() @@ -1238,8 +1195,59 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .orElse(trackerRegistry.getTrackerForCore(coreName, SlaveCoreStatePublisher.class)); } + /** + * Quickly checks if the given name is associated to a master or standalone core. + * + * @param coreName the core name. + * @return true if the name is associated with a master or standalone mode, false otherwise. + */ private boolean isMasterOrStandalone(String coreName) { return trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class) != null; } -} + + /** + * Adds to the returned report an information message alerting the receiver that this core is a slave, + * and therefore the same request should be re-submited to the corresponding master. + * + * @param report the response report. + */ + private void addAlertMessage(NamedList report) + { + report.add( + "WARNING", + "The requested endpoint is not available on the slave. " + + "Please re-submit the same request to the corresponding Master"); + } + + private BiConsumer> apply(SolrParams params, String parameterName, Consumer action) + { + return (parameter, consumer) -> + ofNullable(params.get(parameterName)) + .map(Long::valueOf) + .ifPresent(action); + } + + private Collection coreNames() + { + return notNullOrEmpty(trackerRegistry.getCoreNames()); + } + + /** + * Returns the core name indicated in the request parameters. + * A first attempt is done in order to check if a standard {@link CoreAdminParams#CORE} parameter is in the request. + * If not, the alternative "coreName" parameter name is used. + * + * @param params the request parameters. + * @return the core name specified in the request, null if the parameter is not found. + */ + private String coreName(SolrParams params) + { + return CORE_PARAMETER_NAMES.stream() + .map(params::get) + .filter(Objects::nonNull) + .map(String::trim) + .findFirst() + .orElse(null); + } +} \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java similarity index 98% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java index 8aca5f99c..a3c2b069d 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java @@ -22,12 +22,10 @@ package org.alfresco.solr; import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.httpclient.AuthenticationException; import org.alfresco.service.cmr.repository.datatype.Duration; import org.alfresco.solr.client.Node; import org.alfresco.solr.tracker.*; import org.alfresco.util.CachingDateFormat; -import org.apache.commons.codec.EncoderException; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.json.JSONException; @@ -43,7 +41,7 @@ import static java.util.Optional.ofNullable; /** * Methods taken from AlfrescoCoreAdminHandler that deal with building reports */ -class HandlerReportBuilder +class HandlerReportHelper { static NamedList buildAclReport(AclTracker tracker, Long aclid) throws JSONException { @@ -60,8 +58,7 @@ class HandlerReportBuilder return nr; } - static NamedList buildTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, MetadataTracker tracker, Long txid) - throws AuthenticationException, IOException, JSONException, EncoderException + static NamedList buildTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, MetadataTracker tracker, Long txid) throws JSONException { NamedList nr = new SimpleOrderedMap<>(); nr.add("TXID", txid); @@ -139,7 +136,8 @@ class HandlerReportBuilder } else { - payload.add("WARNING", "This response comes from a slave core. Please consider to ask the same to its corresponding master core, in order to get more information about the requested Node"); + payload.add("WARNING", "This response comes from a slave core and it contains minimal information about the node. " + + "Please consider to re-submit the same request to the corresponding Master, in order to get more information."); } ofNullable(nodeReport.getIndexedNodeDocCount()).ifPresent(value -> payload.add("Indexed Node Doc Count", value)); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java new file mode 100644 index 000000000..2254b9cf0 --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.utils; + +import java.util.Collection; +import java.util.Collections; + +public abstract class Utils +{ + /** + * Returns the same input collection if that is not null, otherwise a new empty collection. + * Provides a safe way for iterating over a returned collection (which could be null). + * + * @param values the collection. + * @param the collection type. + * @return the same input collection if that is not null, otherwise a new empty collection. + */ + public static Collection notNullOrEmpty(Collection values) + { + return values != null ? values : Collections.emptyList(); + } + + /** + * Converts the given input in an Integer, otherwise it returns null. + * + * @param value the numeric string. + * @return the corresponding Integer or null in case the input is NaN. + */ + public static Integer toIntOrNull(String value) + { + try + { + return Integer.valueOf(value); + } + catch(NumberFormatException nfe) + { + return null; + } + } +} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java index 955f365b5..c40e07681 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java @@ -20,15 +20,21 @@ package org.alfresco.solr; import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ALFRESCO_CORE_NAME; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARCHIVE_CORE_NAME; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARG_TXID; import static org.alfresco.solr.AlfrescoCoreAdminHandler.STORE_REF_MAP; import static org.alfresco.solr.AlfrescoCoreAdminHandler.VERSION_CORE_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; @@ -49,6 +55,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.junit.Before; @@ -105,6 +112,81 @@ public class AlfrescoCoreAdminHandlerTest when(req.getParams()).thenReturn(params); } + @Test + public void extractShardsWithEmptyParameter_shouldReturnAnEmptyList() + { + assertTrue(alfrescoCoreAdminHandler.extractShards("", Integer.MAX_VALUE).isEmpty()); + } + + @Test + public void extractShardsWithNullParameter_shouldReturnAnEmptyList() + { + assertTrue(alfrescoCoreAdminHandler.extractShards(null, Integer.MAX_VALUE).isEmpty()); + } + + @Test + public void extractShardsWithOneInvalidShard_shouldReturnAnEmptyList() + { + assertTrue(alfrescoCoreAdminHandler.extractShards("This is an invalid shard id", Integer.MAX_VALUE).isEmpty()); + } + + @Test + public void extractShardsWithOneShards_shouldReturnSingletonList() + { + assertEquals(singletonList(1), alfrescoCoreAdminHandler.extractShards("1", Integer.MAX_VALUE)); + } + + @Test + public void extractShardsWithSeveralValidShards_shouldReturnAllOfThemInTheList() + { + assertEquals(asList(1,5,6,11,23), alfrescoCoreAdminHandler.extractShards("1,5,6,11,23", Integer.MAX_VALUE)); + } + + @Test + public void extractShardsWithSeveralValidShards_shouldReturnOnlyValidIdentifiers() + { + assertEquals(asList(1,5,6,11,23), alfrescoCoreAdminHandler.extractShards("1,5,A,6,xyz,11,BB,23,o01z", Integer.MAX_VALUE)); + } + + @Test + public void extractShardsWithSeveralValidShardsAndLimit_shouldConsiderOnlyShardsLesserThanLimit() + { + assertEquals(asList(1,5,6,11,12), alfrescoCoreAdminHandler.extractShards("1,5,6,11,23,25,99,223,12", 23)); + } + + @Test + public void hasAlfrescoCoreWhenInputIsNull_shouldReturnFalse() + { + assertFalse(alfrescoCoreAdminHandler.hasAlfrescoCore(null)); + } + + @Test + public void hasAlfrescoCoreWhenWeHaveNoCore_shouldReturnFalse() + { + assertFalse(alfrescoCoreAdminHandler.hasAlfrescoCore(emptyList())); + } + + @Test + public void hasAlfrescoCoreWhenDoesntHaveAnyTracker_shouldReturnFalse() + { + when(trackerRegistry.hasTrackersForCore(anyString())).thenReturn(false); + assertFalse(alfrescoCoreAdminHandler.hasAlfrescoCore(emptyList())); + } + + @Test + public void hasAlfrescoCoreWithRegisteredTrackers_shouldReturnTrue() + { + when(trackerRegistry.hasTrackersForCore("CoreD")).thenReturn(true); + assertTrue(alfrescoCoreAdminHandler.hasAlfrescoCore(asList(dummyCore("CoreA"), dummyCore("CoreB"), dummyCore("CoreC"), dummyCore("CoreD")))); + } + + private SolrCore dummyCore(String name) + { + SolrCore core = mock(SolrCore.class); + when(core.getName()).thenReturn(name); + return core; + } + /** Check that a transaction report can be generated. */ @Test public void handleCustomActionTXReportSuccess() throws Exception @@ -212,10 +294,9 @@ public class AlfrescoCoreAdminHandlerTest public void coreNamesAreTrimmed_oneCoreNameAtTime() { AlfrescoCoreAdminHandler spy = spy(new AlfrescoCoreAdminHandler() { @Override - protected boolean newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) + protected void newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) { // Do nothing here otherwise we cannot spy it - return true; } }); @@ -238,10 +319,9 @@ public class AlfrescoCoreAdminHandlerTest public void validAndInvalidCoreNames() { AlfrescoCoreAdminHandler spy = spy(new AlfrescoCoreAdminHandler() { @Override - protected boolean newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) + protected void newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) { // Do nothing here otherwise we cannot spy it - return true; } }); From ba470e8913c847dbc130463ed5c1a38c6dfda09e Mon Sep 17 00:00:00 2001 From: agazzarini Date: Mon, 11 Nov 2019 10:51:11 +0100 Subject: [PATCH 69/76] [ SEARCH-1917 ] better core name parameter handling --- .../solr/AlfrescoCoreAdminHandler.java | 59 +++++++++++++------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index e43d9b536..c0ba58644 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -286,8 +286,11 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler protected void handleCustomAction(SolrQueryRequest req, SolrQueryResponse rsp) { SolrParams params = req.getParams(); - String action = params.get(CoreAdminParams.ACTION); - action = Objects.requireNonNullElse(action.toUpperCase(), ""); + String action = + ofNullable(params.get(CoreAdminParams.ACTION)) + .map(String::trim) + .map(String::toUpperCase) + .orElse(""); try { switch (action) @@ -651,7 +654,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler private void actionCHECK(SolrParams params) { - String cname = params.get(CoreAdminParams.CORE); + String cname = coreName(params); coreNames().stream() .filter(coreName -> cname == null || coreName.equals(cname)) .map(trackerRegistry::getTrackersForCore) @@ -670,8 +673,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler NamedList report = new SimpleOrderedMap<>(); rsp.add(REPORT, report); + String requestedCoreName = coreName(params); + coreNames().stream() - .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(trackerRegistry::hasTrackersForCore) .map(coreName -> new Pair<>(coreName, nodeStatePublisher(coreName))) .filter(coreNameAndPublisher -> coreNameAndPublisher.getSecond() != null) @@ -691,8 +696,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler NamedList report = new SimpleOrderedMap<>(); rsp.add(REPORT, report); + String requestedCoreName = coreName(params); + coreNames().stream() - .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .map(coreName -> new Pair<>(coreName, trackerRegistry.getTrackerForCore(coreName, AclTracker.class))) .filter(coreNameAndAclTracker -> coreNameAndAclTracker.getSecond() != null) .forEach(coreNameAndAclTracker -> @@ -709,7 +716,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler private void actionTXREPORT(SolrQueryResponse rsp, SolrParams params) throws JSONException { String coreName = - ofNullable(params.get(CoreAdminParams.CORE)) + ofNullable(coreName(params)) .orElseThrow(() -> new AlfrescoRuntimeException("No " + params.get(CoreAdminParams.CORE + " parameter set."))); NamedList report = new SimpleOrderedMap<>(); @@ -741,8 +748,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler NamedList report = new SimpleOrderedMap<>(); rsp.add(REPORT, report); + String requestedCoreName = coreName(params); + coreNames().stream() - .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .map(coreName -> new Pair<>(coreName, trackerRegistry.getTrackerForCore(coreName, AclTracker.class))) .filter(coreNameAndAclTracker -> coreNameAndAclTracker.getSecond() != null) .forEach(coreNameAndAclTracker -> @@ -764,7 +773,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler private void rangeCheck(SolrQueryResponse rsp, SolrParams params) throws IOException { String coreName = - ofNullable(params.get(CoreAdminParams.CORE)) + ofNullable(coreName(params)) .orElseThrow(() -> new AlfrescoRuntimeException("No " + params.get(CoreAdminParams.CORE + " parameter set."))); if (isMasterOrStandalone(coreName)) @@ -860,7 +869,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler private synchronized void expand(SolrQueryResponse rsp, SolrParams params) throws IOException { String coreName = - ofNullable(params.get(CoreAdminParams.CORE)) + ofNullable(coreName(params)) .orElseThrow(() -> new AlfrescoRuntimeException("No " + params.get(CoreAdminParams.CORE + " parameter set."))); if (isMasterOrStandalone(coreName)) @@ -944,8 +953,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler Long fromAclTx = getSafeLong(params, "fromAclTx"); Long toAclTx = getSafeLong(params, "toAclTx"); + String requestedCoreName = coreName(params); + coreNames().stream() - .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(trackerRegistry::hasTrackersForCore) .filter(this::isMasterOrStandalone) .forEach(coreName -> @@ -980,8 +991,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .andThen(apply(params, ARG_ACLID, aclTracker::addAclToPurge)); }; + String requestedCoreName = coreName(params); + coreNames().stream() - .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(this::isMasterOrStandalone) .forEach(purgeOnSpecificCore); } @@ -1000,8 +1013,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler ofNullable(params.get(ARG_QUERY)).ifPresent(metadataTracker::addQueryToReindex); }; + String requestedCoreName = coreName(params); + coreNames().stream() - .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(this::isMasterOrStandalone) .forEach(reindexOnSpecificCore); } @@ -1026,8 +1041,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } }; + String requestedCoreName = coreName(params); + coreNames().stream() - .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(this::isMasterOrStandalone) .forEach(retryOnSpecificCore); } @@ -1044,16 +1061,20 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .andThen(apply(params, ARG_ACLID, aclTracker::addAclToIndex)); }; + String requestedCoreName = coreName(params); + coreNames().stream() - .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(this::isMasterOrStandalone) .forEach(indexOnSpecificCore); } private void actionFIX(SolrParams params) throws JSONException { + String requestedCoreName = coreName(params); + coreNames().stream() - .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(this::isMasterOrStandalone) .forEach(this::fixOnSpecificCore); } @@ -1097,8 +1118,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler NamedList report = new SimpleOrderedMap<>(); rsp.add("Summary", report); + String requestedCoreName = coreName(params); + coreNames().stream() - .filter(coreName -> params.get(CoreAdminParams.CORE) == null || coreName.equals(params.get(CoreAdminParams.CORE))) + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(this::isMasterOrStandalone) .forEach(coreName -> coreSummary(params, report, coreName)); } @@ -1220,12 +1243,12 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler "Please re-submit the same request to the corresponding Master"); } - private BiConsumer> apply(SolrParams params, String parameterName, Consumer action) + private BiConsumer> apply(SolrParams params, String parameterName, Consumer executeSideEffectAction) { return (parameter, consumer) -> ofNullable(params.get(parameterName)) .map(Long::valueOf) - .ifPresent(action); + .ifPresent(executeSideEffectAction); } private Collection coreNames() From 7f9e5e193dec0648c958bf3390df69c7fae517eb Mon Sep 17 00:00:00 2001 From: agazzarini Date: Mon, 11 Nov 2019 13:10:43 +0100 Subject: [PATCH 70/76] [ SEARCH-1917 ] Working implementation + Unit tests --- .../solr/AlfrescoCoreAdminHandler.java | 67 ++++++------ .../solr/AlfrescoCoreAdminHandlerTest.java | 101 ++++++++++++++++-- 2 files changed, 125 insertions(+), 43 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index c0ba58644..3aea3584f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -70,7 +70,6 @@ import java.util.Objects; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -593,8 +592,6 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .anyMatch(trackerRegistry::hasTrackersForCore); } - // ::::: - private void updateShared(SolrQueryRequest req) { SolrParams params = req.getParams(); @@ -604,7 +601,9 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler File config = new File(AlfrescoSolrDataModel.getResourceDirectory(), AlfrescoSolrDataModel.SHARED_PROPERTIES); updateSharedProperties(params, config, hasAlfrescoCore(coreContainer.getCores())); - coreContainer.getCores().forEach(aCore -> coreContainer.reload(aCore.getName())); + coreContainer.getCores().stream() + .map(SolrCore::getName) + .forEach(coreContainer::reload); } catch (IOException e) { @@ -678,7 +677,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler coreNames().stream() .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(trackerRegistry::hasTrackersForCore) - .map(coreName -> new Pair<>(coreName, nodeStatePublisher(coreName))) + .map(coreName -> new Pair<>(coreName, coreStatePublisher(coreName))) .filter(coreNameAndPublisher -> coreNameAndPublisher.getSecond() != null) .forEach(coreNameAndPublisher -> report.add( @@ -985,10 +984,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - apply(params, ARG_TXID, metadataTracker::addTransactionToPurge) - .andThen(apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToPurge)) - .andThen(apply(params, ARG_NODEID, metadataTracker::addNodeToPurge)) - .andThen(apply(params, ARG_ACLID, aclTracker::addAclToPurge)); + apply(params, ARG_TXID, metadataTracker::addTransactionToPurge); + apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToPurge); + apply(params, ARG_NODEID, metadataTracker::addNodeToPurge); + apply(params, ARG_ACLID, aclTracker::addAclToPurge); }; String requestedCoreName = coreName(params); @@ -1005,10 +1004,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - apply(params, ARG_TXID, metadataTracker::addTransactionToReindex) - .andThen(apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToReindex)) - .andThen(apply(params, ARG_NODEID, metadataTracker::addNodeToReindex)) - .andThen(apply(params, ARG_ACLID, aclTracker::addAclToReindex)); + apply(params, ARG_TXID, metadataTracker::addTransactionToReindex); + apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToReindex); + apply(params, ARG_NODEID, metadataTracker::addNodeToReindex); + apply(params, ARG_ACLID, aclTracker::addAclToReindex); ofNullable(params.get(ARG_QUERY)).ifPresent(metadataTracker::addQueryToReindex); }; @@ -1055,10 +1054,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - apply(params, ARG_TXID, metadataTracker::addTransactionToIndex) - .andThen(apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToIndex)) - .andThen(apply(params, ARG_NODEID, metadataTracker::addNodeToIndex)) - .andThen(apply(params, ARG_ACLID, aclTracker::addAclToIndex)); + apply(params, ARG_TXID, metadataTracker::addTransactionToIndex); + apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToIndex); + apply(params, ARG_NODEID, metadataTracker::addNodeToIndex); + apply(params, ARG_ACLID, aclTracker::addAclToIndex); }; String requestedCoreName = coreName(params); @@ -1113,7 +1112,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } - private void actionSUMMARY(SolrQueryResponse rsp, SolrParams params) + void actionSUMMARY(SolrQueryResponse rsp, SolrParams params) { NamedList report = new SimpleOrderedMap<>(); rsp.add("Summary", report); @@ -1122,7 +1121,6 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler coreNames().stream() .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) - .filter(this::isMasterOrStandalone) .forEach(coreName -> coreSummary(params, report, coreName)); } @@ -1162,13 +1160,9 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } - private DocRouter getDocRouter(String cname) + DocRouter getDocRouter(String cname) { - return notNullOrEmpty(trackerRegistry.getTrackersForCore(cname)) - .stream() - .filter(tracker -> tracker instanceof MetadataTracker) - .findAny() - .map(MetadataTracker.class::cast) + return ofNullable(trackerRegistry.getTrackerForCore(cname, MetadataTracker.class)) .map(MetadataTracker::getDocRouter) .orElse(null); } @@ -1211,7 +1205,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * @param coreName the owning core name. * @return the component which is in charge to publish the core state. */ - private CoreStatePublisher nodeStatePublisher(String coreName) + CoreStatePublisher coreStatePublisher(String coreName) { return ofNullable(trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class)) .map(CoreStatePublisher.class::cast) @@ -1224,7 +1218,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * @param coreName the core name. * @return true if the name is associated with a master or standalone mode, false otherwise. */ - private boolean isMasterOrStandalone(String coreName) + boolean isMasterOrStandalone(String coreName) { return trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class) != null; } @@ -1243,19 +1237,18 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler "Please re-submit the same request to the corresponding Master"); } - private BiConsumer> apply(SolrParams params, String parameterName, Consumer executeSideEffectAction) - { - return (parameter, consumer) -> - ofNullable(params.get(parameterName)) - .map(Long::valueOf) - .ifPresent(executeSideEffectAction); - } - - private Collection coreNames() + Collection coreNames() { return notNullOrEmpty(trackerRegistry.getCoreNames()); } + private void apply(SolrParams params, String parameterName, Consumer executeSideEffectAction) + { + ofNullable(params.get(parameterName)) + .map(Long::valueOf) + .ifPresent(executeSideEffectAction); + } + /** * Returns the core name indicated in the request parameters. * A first attempt is done in order to check if a standard {@link CoreAdminParams#CORE} parameter is in the request. @@ -1264,7 +1257,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * @param params the request parameters. * @return the core name specified in the request, null if the parameter is not found. */ - private String coreName(SolrParams params) + String coreName(SolrParams params) { return CORE_PARAMETER_NAMES.stream() .map(params::get) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java index c40e07681..6470a0993 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java @@ -29,6 +29,8 @@ import static org.alfresco.solr.AlfrescoCoreAdminHandler.STORE_REF_MAP; import static org.alfresco.solr.AlfrescoCoreAdminHandler.VERSION_CORE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -48,11 +50,15 @@ import java.util.stream.Collectors; import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.solr.adapters.IOpenBitSet; import org.alfresco.solr.tracker.AclTracker; +import org.alfresco.solr.tracker.DocRouter; import org.alfresco.solr.tracker.IndexHealthReport; import org.alfresco.solr.tracker.MetadataTracker; +import org.alfresco.solr.tracker.PropertyRouter; +import org.alfresco.solr.tracker.SlaveCoreStatePublisher; import org.alfresco.solr.tracker.TrackerRegistry; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; @@ -169,7 +175,6 @@ public class AlfrescoCoreAdminHandlerTest @Test public void hasAlfrescoCoreWhenDoesntHaveAnyTracker_shouldReturnFalse() { - when(trackerRegistry.hasTrackersForCore(anyString())).thenReturn(false); assertFalse(alfrescoCoreAdminHandler.hasAlfrescoCore(emptyList())); } @@ -180,6 +185,95 @@ public class AlfrescoCoreAdminHandlerTest assertTrue(alfrescoCoreAdminHandler.hasAlfrescoCore(asList(dummyCore("CoreA"), dummyCore("CoreB"), dummyCore("CoreC"), dummyCore("CoreD")))); } + @Test + public void trackerRegistryHasNoCoreNames_itShouldReturnAnEmptyList() + { + assertTrue(alfrescoCoreAdminHandler.coreNames().isEmpty()); + } + + @Test + public void coreDetectedAsMasterOrStandalone() + { + MetadataTracker coreStatePublisher = mock(MetadataTracker.class); + + when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))) + .thenReturn(coreStatePublisher); + + assertTrue(alfrescoCoreAdminHandler.isMasterOrStandalone("ThisIsTheCoreName")); + } + + @Test + public void coreDetectedAsSlave() + { + when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))).thenReturn(null); + assertFalse(alfrescoCoreAdminHandler.isMasterOrStandalone("ThisIsTheCoreName")); + } + + @Test + public void coreIsMaster_thenCoreStatePublisherInstanceCorrespondsToMetadataTracker() + { + MetadataTracker coreStatePublisher = mock(MetadataTracker.class); + + when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))) + .thenReturn(coreStatePublisher); + + assertSame(coreStatePublisher, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName")); + } + + @Test + public void coreIsSlave_thenCoreStatePublisherInstanceCorrespondsToSlaveCoreStatePublisher() + { + SlaveCoreStatePublisher coreStatePublisher = mock(SlaveCoreStatePublisher.class); + + when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))).thenReturn(null); + when(trackerRegistry.getTrackerForCore(anyString(), eq(SlaveCoreStatePublisher.class))).thenReturn(coreStatePublisher); + + assertSame(coreStatePublisher, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName")); + } + + @Test + public void coreIsSlave_thenDocRouterIsNull() + { + String coreName = "aCore"; + when(trackerRegistry.getTrackerForCore(eq(coreName), eq(MetadataTracker.class))).thenReturn(null); + assertNull(alfrescoCoreAdminHandler.getDocRouter("aCore")); + } + + @Test + public void coreIsMaster_thenDocRouterIsProperlyReturned() + { + DocRouter expectedRouter = new PropertyRouter("someProperty_.{1,35}"); + + MetadataTracker coreStatePublisher = mock(MetadataTracker.class); + when(coreStatePublisher.getDocRouter()).thenReturn(expectedRouter); + when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))).thenReturn(coreStatePublisher); + + assertSame(expectedRouter, alfrescoCoreAdminHandler.getDocRouter("aCore")); + } + + @Test + public void targetCoreNameCanBeSpecifiedInSeveralWays() + { + String coreName = "ThisIsTheCoreName"; + + ModifiableSolrParams params = new ModifiableSolrParams(); + + assertNull(alfrescoCoreAdminHandler.coreName(params)); + + params.set(CoreAdminParams.CORE, coreName); + + assertEquals(coreName, alfrescoCoreAdminHandler.coreName(params)); + + params.remove(CoreAdminParams.CORE); + assertNull(alfrescoCoreAdminHandler.coreName(params)); + + params.set("coreName", coreName); + + assertEquals(coreName, alfrescoCoreAdminHandler.coreName(params)); + assertEquals(coreName, alfrescoCoreAdminHandler.coreName(params)); + } + + private SolrCore dummyCore(String name) { SolrCore core = mock(SolrCore.class); @@ -225,8 +319,6 @@ public class AlfrescoCoreAdminHandlerTest public void handleCustomActionTXReportMissingTXId() { when(params.get(CoreAdminParams.ACTION)).thenReturn(TXREPORT); - when(params.get(ARG_TXID)).thenReturn(null); - alfrescoCoreAdminHandler.handleCustomAction(req, rsp); verify(rsp, never()).add(anyString(), any()); @@ -238,11 +330,8 @@ public class AlfrescoCoreAdminHandlerTest { when(params.get(CoreAdminParams.ACTION)).thenReturn(TXREPORT); when(params.get(CoreAdminParams.CORE)).thenReturn(null); - when(params.get(ARG_TXID)).thenReturn(TX_ID); alfrescoCoreAdminHandler.handleCustomAction(req, rsp); - - verify(rsp, never()).add(anyString(), any()); } /** Check that when an unknown action is provided we don't generate a report. */ From 2ad47af4f8d8ff2c9d8dc240e8a42c34370e0da9 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Mon, 11 Nov 2019 14:53:42 +0100 Subject: [PATCH 71/76] [ SEARCH-1917 ] Minor fixes --- .../java/org/alfresco/solr/AlfrescoCoreAdminHandler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index 3aea3584f..7255a8d65 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -716,7 +716,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler { String coreName = ofNullable(coreName(params)) - .orElseThrow(() -> new AlfrescoRuntimeException("No " + params.get(CoreAdminParams.CORE + " parameter set."))); + .orElseThrow(() -> new AlfrescoRuntimeException("No " + CoreAdminParams.CORE + " parameter set.")); NamedList report = new SimpleOrderedMap<>(); rsp.add(REPORT, report); @@ -773,7 +773,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler { String coreName = ofNullable(coreName(params)) - .orElseThrow(() -> new AlfrescoRuntimeException("No " + params.get(CoreAdminParams.CORE + " parameter set."))); + .orElseThrow(() -> new AlfrescoRuntimeException("No " + CoreAdminParams.CORE + " parameter set.")); if (isMasterOrStandalone(coreName)) { @@ -869,7 +869,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler { String coreName = ofNullable(coreName(params)) - .orElseThrow(() -> new AlfrescoRuntimeException("No " + params.get(CoreAdminParams.CORE + " parameter set."))); + .orElseThrow(() -> new AlfrescoRuntimeException("No " + CoreAdminParams.CORE + " parameter set.")); if (isMasterOrStandalone(coreName)) { From 08deb79b5ee57d748362854709c2555b96b6b654 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Tue, 12 Nov 2019 08:20:54 +0100 Subject: [PATCH 72/76] [ SEARCH-1917 ] Review comments addressed --- .../src/main/java/org/alfresco/solr/SolrInformationServer.java | 3 ++- .../src/main/java/org/alfresco/solr/content/AccessMode.java | 2 +- .../src/main/java/org/alfresco/solr/content/ChangeSet.java | 2 +- .../org/alfresco/solr/content/InitialisableAccessMode.java | 2 +- .../main/java/org/alfresco/solr/content/SolrContentStore.java | 2 +- .../java/org/alfresco/solr/content/SolrContentUrlBuilder.java | 2 +- .../java/org/alfresco/solr/content/SolrFileContentReader.java | 2 +- .../java/org/alfresco/solr/content/SolrFileContentWriter.java | 2 +- .../src/main/java/org/alfresco/solr/content/package-info.java | 2 +- 9 files changed, 10 insertions(+), 9 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 1f9b643fe..189a5cf8f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -3894,7 +3894,8 @@ public class SolrInformationServer implements InformationServer } @Override - public void flushContentStore() throws IOException { + public void flushContentStore() throws IOException + { solrContentStore.flushChangeSet(); } } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/AccessMode.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/AccessMode.java index 468526a10..42873d17b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/AccessMode.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/AccessMode.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2014 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java index 95180f059..edb9c7482 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2016 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/InitialisableAccessMode.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/InitialisableAccessMode.java index f8fa1e9a8..78a0fea2e 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/InitialisableAccessMode.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/InitialisableAccessMode.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2014 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java index acf472650..a11f3f4ba 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2014 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentUrlBuilder.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentUrlBuilder.java index 542636111..36b38b07b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentUrlBuilder.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentUrlBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2014 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentReader.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentReader.java index 864e973d7..c6c7e2a14 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentReader.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentReader.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2014 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentWriter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentWriter.java index bb3dbc142..2a5a05013 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentWriter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentWriter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2014 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/package-info.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/package-info.java index ac6d0aec4..9fd22aca6 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/package-info.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2016 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * From ed7208aa0f8e8595fdab9021ed7180c6305eef9b Mon Sep 17 00:00:00 2001 From: agazzarini Date: Tue, 12 Nov 2019 08:23:36 +0100 Subject: [PATCH 73/76] [ SEARCH-1858 ] minor fix on logging typo --- .../src/main/java/org/alfresco/solr/content/ChangeSet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java index edb9c7482..21eee54d5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java @@ -263,7 +263,7 @@ public class ChangeSet implements AutoCloseable if (adds.isEmpty() && deletes.isEmpty()) { - LOGGER.debug("no changes in contentstore to flush"); + LOGGER.debug("No changes in contentstore to flush"); return; } From 8c90303a6d1ac6a65c68c342dbf4c2d765b8fc67 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 12 Nov 2019 16:26:50 +0100 Subject: [PATCH 74/76] [contentStoreReplication] cleanup SOLRAPIQueueClient data structures after contentStoreReplicationIT. Random solrhomes for distributed tests. --- .../solr/AbstractAlfrescoDistributedIT.java | 1 + .../org/alfresco/solr/SolrITInitializer.java | 13 +++++--- .../handler/ContentStoreReplicationIT.java | 33 ++++++++++++++----- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java index 33a0761e4..3f6cdffe9 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java @@ -296,6 +296,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer { long begin = System.currentTimeMillis(); //TODO: Support multiple cores per jetty + System.out.println(getStandaloneClients().size()); SolrClient standaloneClient = getStandaloneClients().get(0); //Get the first one waitForDocCountCore(standaloneClient, query, count, waitMillis, begin); waitForShardsCount(query, count, waitMillis, begin); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index 7b21f7878..92bcc56a9 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -4,6 +4,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; import org.alfresco.solr.basics.RandomSupplier; import org.alfresco.solr.client.SOLRAPIQueueClient; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.util.Time; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.embedded.JettyConfig; @@ -319,8 +320,10 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 { boolean basicAuth = additionalProperties != null ? Boolean.parseBoolean(additionalProperties.getProperty("BasicAuth", "false")) : false; - JettySolrRunner jsr = createJetty(jettyKey, basicAuth); - jettyContainers.put(jettyKey, jsr); + String mainKey = jettyKey + "main_" + Time.now() + "/solrhome"; + + JettySolrRunner jsr = createJetty(mainKey, basicAuth); + jettyContainers.put(mainKey, jsr); Properties properties = new Properties(); @@ -331,7 +334,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 for (int i = 0; i < coreNames.length; i++) { - addCoreToJetty(jettyKey, coreNames[i], coreNames[i], properties); + addCoreToJetty(mainKey, coreNames[i], coreNames[i], properties); } //Now start jetty @@ -366,7 +369,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 props.put("shard.range", ranges[i]); } - String shardKey = jettyKey+"_shard_"+i+"/solrhome"; + String shardKey = jettyKey+"_shard_" + i + "_" + Time.now() + "/solrhome"; JettySolrRunner j = createJetty(shardKey, basicAuth); //use the first corename specified as the Share template addCoreToJetty(shardKey, coreNames[0], shardname, props); @@ -384,7 +387,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 protected static void destroyServers() throws Exception { - List solrHomes = new ArrayList(); + List solrHomes = new ArrayList<>(); for (JettySolrRunner jetty : jettyContainers.values()) { solrHomes.add(jetty.getSolrHome()); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java index d903b3974..8fb8eeb58 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java @@ -24,8 +24,10 @@ import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; import org.alfresco.solr.client.Node; import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.SOLRAPIQueueClient; import org.alfresco.solr.client.Transaction; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.util.Time; import org.apache.lucene.index.Term; import org.apache.lucene.search.TermQuery; import org.apache.solr.SolrTestCaseJ4; @@ -94,8 +96,11 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT String coreName = "master"; boolean basicAuth = Boolean.parseBoolean(properties.getProperty("BasicAuth", "false")); - String masterKey = "master/solrHome"; - String slaveKey = "slave/solrHome"; + String masterDir = "master" + Time.now(); + String slaveDir = "slave" + Time.now(); + + String masterKey = masterDir + "/solrHome"; + String slaveKey = slaveDir + "/solrHome"; master = createJetty(masterKey, basicAuth); addCoreToJetty(masterKey, coreName, coreName, null); @@ -117,8 +122,8 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT masterSolrHome = testDir.toPath().resolve(masterKey); slaveSolrHome = testDir.toPath().resolve(slaveKey); - masterContentStore = testDir.toPath().resolve("master/contentstore"); - slaveContentStore = testDir.toPath().resolve("slave/contentstore"); + masterContentStore = testDir.toPath().resolve(masterDir + "/contentstore"); + slaveContentStore = testDir.toPath().resolve(slaveDir + "/contentstore"); AclChangeSet aclChangeSet = getAclChangeSet(1); @@ -132,16 +137,26 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT @AfterClass - public static void cleanupMasterSlave() throws Exception { + public static void cleanupMasterSlave() throws Exception + { master.stop(); slave.stop(); FileUtils.forceDelete(new File(masterSolrHome.getParent().toUri())); FileUtils.forceDelete(new File(slaveSolrHome.getParent().toUri())); + + SOLRAPIQueueClient.nodeMetaDataMap.clear(); + SOLRAPIQueueClient.transactionQueue.clear(); + SOLRAPIQueueClient.aclChangeSetQueue.clear(); + SOLRAPIQueueClient.aclReadersMap.clear(); + SOLRAPIQueueClient.aclMap.clear(); + SOLRAPIQueueClient.nodeMap.clear(); + SOLRAPIQueueClient.nodeContentMap.clear(); } @Test - public void contentStoreReplicationTest() throws Exception { + public void contentStoreReplicationTest() throws Exception + { // ADD 250 nodes and check they are replicated int numNodes = 250; Transaction bigTxn = getTransaction(0, numNodes); @@ -231,7 +246,8 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT } - private static boolean waitForContentStoreSync(long waitMillis) throws InterruptedException { + private static boolean waitForContentStoreSync(long waitMillis) throws InterruptedException + { long startMillis = System.currentTimeMillis(); long timeout = startMillis + waitMillis; @@ -254,7 +270,8 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT return false; } - private static void setMasterUrl(String jettyKey, String coreName, String masterUrl) throws IOException { + private static void setMasterUrl(String jettyKey, String coreName, String masterUrl) throws IOException + { Path jettySolrHome = testDir.toPath().resolve(jettyKey); Path coreHome = jettySolrHome.resolve(coreName); Path confDir = coreHome.resolve("conf"); From b0479866205feba7aa58b3db74e08fb59962307b Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 12 Nov 2019 18:58:06 +0100 Subject: [PATCH 75/76] [contentStoreReplication] create jetty now takes as parameters solrhome and jettykey --- .../org/alfresco/solr/SolrITInitializer.java | 26 ++++++------------- .../handler/ContentStoreReplicationIT.java | 4 +-- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index 92bcc56a9..8aa434e16 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -227,23 +227,13 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 SOLRAPIQueueClient.nodeMap.clear(); } - - /** - * Creates a JettySolrRunner (if one didn't exist already). DOES NOT START IT. - * @return - * @throws Exception - */ - protected static JettySolrRunner createJetty(String jettyKey, boolean basicAuth) throws Exception - { - return createJetty(jettyKey, basicAuth, 0); - } /** * Creates a JettySolrRunner (if one didn't exist already). DOES NOT START IT. * @return * @throws Exception */ - protected static JettySolrRunner createJetty(String jettyKey, boolean basicAuth, int port) throws Exception + protected static JettySolrRunner createJetty(String jettyKey, String solrhome, boolean basicAuth) throws Exception { if (jettyContainers.containsKey(jettyKey)) { @@ -251,9 +241,9 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 } else { - Path jettySolrHome = testDir.toPath().resolve(jettyKey); + Path jettySolrHome = testDir.toPath().resolve(solrhome); seedSolrHome(jettySolrHome); - JettySolrRunner jetty = createJetty(jettySolrHome.toFile(), null, null, false, port, getSchemaFile(), basicAuth); + JettySolrRunner jetty = createJetty(jettySolrHome.toFile(), null, null, false, 0, getSchemaFile(), basicAuth); return jetty; } } @@ -320,10 +310,10 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 { boolean basicAuth = additionalProperties != null ? Boolean.parseBoolean(additionalProperties.getProperty("BasicAuth", "false")) : false; - String mainKey = jettyKey + "main_" + Time.now() + "/solrhome"; + String solrHome = jettyKey + "main_" + Time.now() + "/solrhome"; - JettySolrRunner jsr = createJetty(mainKey, basicAuth); - jettyContainers.put(mainKey, jsr); + JettySolrRunner jsr = createJetty(jettyKey, solrHome, basicAuth); + jettyContainers.put(jettyKey, jsr); Properties properties = new Properties(); @@ -334,7 +324,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 for (int i = 0; i < coreNames.length; i++) { - addCoreToJetty(mainKey, coreNames[i], coreNames[i], properties); + addCoreToJetty(solrHome, coreNames[i], coreNames[i], properties); } //Now start jetty @@ -370,7 +360,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 } String shardKey = jettyKey+"_shard_" + i + "_" + Time.now() + "/solrhome"; - JettySolrRunner j = createJetty(shardKey, basicAuth); + JettySolrRunner j = createJetty(shardKey, shardKey, basicAuth); //use the first corename specified as the Share template addCoreToJetty(shardKey, coreNames[0], shardname, props); solrShards.add(j); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java index 8fb8eeb58..52bce2fdd 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java @@ -102,13 +102,13 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT String masterKey = masterDir + "/solrHome"; String slaveKey = slaveDir + "/solrHome"; - master = createJetty(masterKey, basicAuth); + master = createJetty("master", masterKey, basicAuth); addCoreToJetty(masterKey, coreName, coreName, null); startJetty(master); String slaveCoreName = "slave"; - slave = createJetty(slaveKey, basicAuth); + slave = createJetty("slave", slaveKey, basicAuth); addCoreToJetty(slaveKey, slaveCoreName, slaveCoreName, null); setMasterUrl(slaveKey, slaveCoreName, master.getBaseUrl().toString() + "/master"); From 1ed6412cd53d6ba38f63dbde83686466f3a18170 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Wed, 13 Nov 2019 11:28:18 +0100 Subject: [PATCH 76/76] [contentStoreReplicationr] create jetty fixed solrhome settings --- .../java/org/alfresco/solr/SolrITInitializer.java | 13 ++++++------- .../solr/handler/ContentStoreReplicationIT.java | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index 8aa434e16..b06df1a09 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -233,7 +233,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 * @return * @throws Exception */ - protected static JettySolrRunner createJetty(String jettyKey, String solrhome, boolean basicAuth) throws Exception + protected static JettySolrRunner createJetty(String jettyKey, boolean basicAuth) throws Exception { if (jettyContainers.containsKey(jettyKey)) { @@ -241,7 +241,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 } else { - Path jettySolrHome = testDir.toPath().resolve(solrhome); + Path jettySolrHome = testDir.toPath().resolve(jettyKey); seedSolrHome(jettySolrHome); JettySolrRunner jetty = createJetty(jettySolrHome.toFile(), null, null, false, 0, getSchemaFile(), basicAuth); return jetty; @@ -310,9 +310,8 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 { boolean basicAuth = additionalProperties != null ? Boolean.parseBoolean(additionalProperties.getProperty("BasicAuth", "false")) : false; - String solrHome = jettyKey + "main_" + Time.now() + "/solrhome"; - JettySolrRunner jsr = createJetty(jettyKey, solrHome, basicAuth); + JettySolrRunner jsr = createJetty(jettyKey, basicAuth); jettyContainers.put(jettyKey, jsr); Properties properties = new Properties(); @@ -324,7 +323,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 for (int i = 0; i < coreNames.length; i++) { - addCoreToJetty(solrHome, coreNames[i], coreNames[i], properties); + addCoreToJetty(jettyKey, coreNames[i], coreNames[i], properties); } //Now start jetty @@ -359,8 +358,8 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 props.put("shard.range", ranges[i]); } - String shardKey = jettyKey+"_shard_" + i + "_" + Time.now() + "/solrhome"; - JettySolrRunner j = createJetty(shardKey, shardKey, basicAuth); + String shardKey = jettyKey+"_shard_" + i + "/solrhome"; + JettySolrRunner j = createJetty(shardKey, basicAuth); //use the first corename specified as the Share template addCoreToJetty(shardKey, coreNames[0], shardname, props); solrShards.add(j); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java index 52bce2fdd..8fb8eeb58 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java @@ -102,13 +102,13 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT String masterKey = masterDir + "/solrHome"; String slaveKey = slaveDir + "/solrHome"; - master = createJetty("master", masterKey, basicAuth); + master = createJetty(masterKey, basicAuth); addCoreToJetty(masterKey, coreName, coreName, null); startJetty(master); String slaveCoreName = "slave"; - slave = createJetty("slave", slaveKey, basicAuth); + slave = createJetty(slaveKey, basicAuth); addCoreToJetty(slaveKey, slaveCoreName, slaveCoreName, null); setMasterUrl(slaveKey, slaveCoreName, master.getBaseUrl().toString() + "/master");