Merge branch 'master' of https://git.alfresco.com/search_discovery/insightengine into feature/SEARCH-1881_enhanceSpellcheckTests

This commit is contained in:
Keerat
2019-11-15 13:19:04 +00:00
51 changed files with 10617 additions and 434 deletions
+39
View File
@@ -179,6 +179,45 @@
<directory>src/main/resources/solr/instance/templates/rerank/conf</directory>
<excludes>
<exclude>solrconfig.xml</exclude>
<exclude>solrcore.properties</exclude>
</excludes>
</resource>
</resources>
</configuration>
</execution>
<execution>
<id>copy-production-solr-configuration-for-master</id>
<phase>generate-test-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.testOutputDirectory}/test-files/master/conf</outputDirectory>
<resources>
<resource>
<directory>src/main/resources/solr/instance/templates/rerank/conf</directory>
<excludes>
<exclude>solrconfig.xml</exclude>
<exclude>solrcore.properties</exclude>
</excludes>
</resource>
</resources>
</configuration>
</execution>
<execution>
<id>copy-production-solr-configuration-for-slave</id>
<phase>generate-test-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.testOutputDirectory}/test-files/slave/conf</outputDirectory>
<resources>
<resource>
<directory>src/main/resources/solr/instance/templates/rerank/conf</directory>
<excludes>
<exclude>solrconfig.xml</exclude>
<exclude>solrcore.properties</exclude>
</excludes>
</resource>
</resources>
@@ -25,6 +25,7 @@ import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.solr.adapters.IOpenBitSet;
import org.alfresco.solr.client.SOLRAPIClientFactory;
import org.alfresco.solr.config.ConfigUtil;
import org.alfresco.solr.content.SolrContentStore;
import org.alfresco.solr.tracker.AclTracker;
import org.alfresco.solr.tracker.CoreStatePublisher;
import org.alfresco.solr.tracker.DBIDRangeRouter;
@@ -146,6 +147,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
private ConcurrentHashMap<String, InformationServer> informationServers;
private static List<String> CORE_PARAMETER_NAMES = asList(CoreAdminParams.CORE, "coreName", "index");
private SolrContentStore contentStore;
public AlfrescoCoreAdminHandler()
{
@@ -161,6 +163,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
trackerRegistry = new TrackerRegistry();
informationServers = new ConcurrentHashMap<>();
this.scheduler = new SolrTrackerScheduler(this);
if (coreContainer != null)
{
this.contentStore = new SolrContentStore(coreContainer.getSolrHome());
}
String createDefaultCores = ConfigUtil.locateProperty(ALFRESCO_DEFAULTS, "");
int numShards = Integer.parseInt(ConfigUtil.locateProperty(NUM_SHARDS, "1"));
@@ -249,15 +255,28 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
{
scheduler.pauseAll();
if (trackerRegistry.getModelTracker() != null) trackerRegistry.getModelTracker().shutdown();
if (trackerRegistry.getModelTracker() != null)
trackerRegistry.getModelTracker().shutdown();
trackerRegistry.setModelTracker(null);
scheduler.shutdown();
}
}
catch(Exception exception)
catch (Exception exception)
{
LOGGER.error("Problem shutting down Alfresco core container services", exception);
LOGGER.error(
"Unable to properly shut down Alfresco core container services. See the exception below for further details.",
exception);
}
try
{
contentStore.close();
}
catch (Exception exception)
{
LOGGER.error("Unable to properly shut down the ContentStore. See the exception below for further details.",
exception);
}
}
@@ -1266,4 +1285,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
.findFirst()
.orElse(null);
}
public SolrContentStore getSolrContentStore()
{
return contentStore;
}
}
@@ -179,4 +179,6 @@ public interface InformationServer extends InformationServerCollectionProvider
String getHostName();
String getBaseUrl();
void flushContentStore() throws IOException;
}
@@ -3892,4 +3892,10 @@ public class SolrInformationServer implements InformationServer
}
return searchers;
}
@Override
public void flushContentStore() throws IOException
{
solrContentStore.flushChangeSet();
}
}
@@ -0,0 +1,106 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
import org.alfresco.solr.client.NodeMetaData;
import org.apache.solr.common.SolrInputDocument;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* Behavioural interface which indicates the type of content store access mode for the owning node.
* A replication interaction is composed at least by 2 roles: a master and a slave.
*
* Despite the same interface, the behaviour of the content store management changes depending on the node kind:
*
* <ul>
* <li>Master Node: READ + WRITE + Changes tracking. Writes and changes tracking are a direct consequence of the "indexing" nature of the master node.</li>
* <li>Slave Node: READ ONLY (i.e. never write: changes are applied on the master and replicated on slaves)</li>
* </ul>
*
* Important: the owning entity *is not* the SolrCore instance: the Content store is shared across all cores of
* A node can transit from Read to Write mode
*
* @author Andrea Gazzarini
* @since 1.5
*/
interface AccessMode extends Closeable
{
/**
* Returns the last persisted content store version.
*
* @return the last persisted content store version, SolrContentStore#NO_VERSION_AVAILABLE in case the version isn't available.
*/
long getLastCommittedVersion();
/**
* Persists the last committed version on the hosting node.
* Note that this is tipically valid only on slave node, because the master already manages the content store version on
* a persistent storage, so it doesn't need to call this method.
*
* @param version the last committed content store version.
*/
void setLastCommittedVersion(long version);
Map<String, List<Map<String, Object>>> getChanges(long version);
/**
* Stores a {@link SolrInputDocument} into Alfresco solr content store.
*
* @param tenant the owning tenant.
* @param dbId the document DBID
* @param doc the document itself.
*/
void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc);
/**
* Stores a {@link SolrInputDocument} into Alfresco solr content store.
*
* @param nodeMetaData the node metadata.
* @param doc the document itself.
*/
void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc);
/**
* Removes a node from the content store.
*
* @param nodeMetaData the node metadata.
*/
void removeDocFromContentStore(NodeMetaData nodeMetaData);
/**
* Flushes pending changesets.
*
* @throws IOException in case of I/O failure.
*/
void flushChangeSet() throws IOException;
/**
* Tries to transition from this mode to the readOnlyMode.
*/
void switchOnReadOnlyMode();
/**
* Tries to transition from this mode to the read/write mode.
*/
void switchOnReadWriteMode();
}
@@ -0,0 +1,460 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
import static java.util.Arrays.asList;
import static java.util.Arrays.stream;
import static java.util.Collections.emptySet;
import static java.util.Optional.ofNullable;
import org.alfresco.util.Pair;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedNumericSortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.BinaryOperator;
/**
* Stores and manages changes occurred in a content store.
*
* Note this entity is part of the content store only when it is set on writable mode (i.e. only when the hosting node
* is a standalone instance or it is a master node).
*
* That means if the hosting node has at least one standalone core or one master node, then the managed content store
* will be set in write mode and therefore a {@link ChangeSet} instance will track all changes applied to it.
*
* In that mode, each change is associated to a given (incremental) version; changes consist of adds/updates and deletes.
* Each time the {@link org.alfresco.solr.tracker.CommitTracker} executes a commit in the main index, all content store
* changes accumulated in the meantime in the current {@link ChangeSet} instance are flushed in an auxiliary Lucene index
* and therefore persisted.
*
* On the other side, when all cores on a given node are configured as slaves, the content store is in read-only mode,
* and there's no need to track any change (i.e. changes are coming from master through the replication procedure).
*
* As a side note: we can have a persistent or transient {@link ChangeSet} instance. The first one is used to manage, as
* the name suggests, in a persistent way the content store changes accumulated during the indexing operations (see the
* description above about its behaviour in master nodes).
*
* The second one is used instead for computing/reducing the "merged" list of changes that we need to communicate to slave nodes:
* as part of the replication mechanism, a slave communicates its last synched content store version; on the master side,
* we need to compute and communicate back all changes that the slave needs to apply since that version in order to be in
* synch with master.
*
* @author Andrea Gazzarini
* @since 1.5
*/
public class ChangeSet implements AutoCloseable
{
private final static Logger LOGGER = LoggerFactory.getLogger(ChangeSet.class);
private final static ChangeSet EMPTY_CHANGESET = new ChangeSet.Builder().empty().build();
/**
* Builds class for creating {@link ChangeSet} instances.
* The builder is here for creating three kind of {@link ChangeSet} instances:
*
* <ul>
* <li>Persistent: used for tracking and persisting content store changes.</li>
* <li>Transient: used for computing the merged list of changes a slave should apply for being in synch with master.</li>
* <li>Empty: an immutable "NullObject" used for denoting an empty {@link ChangeSet} instance.</li>
* </ul>
*
* @see <a href="https://en.wikipedia.org/wiki/Null_object_pattern">Null Object Design Pattern</a>
*/
public static class Builder
{
private String root;
private boolean immutable;
/**
* Adds a content store root folder to this builder.
* This automatically implies we are creating a persistent {@link ChangeSet} instance.
*
* @param root the absolute path of the content store root folder.
* @return this builder, for being used in a fluent mode.
*/
Builder withContentStoreRoot(String root)
{
if (root == null) throw new IllegalArgumentException("Unable to build the Changeset structures with a null content store root folder.");
if (!new File(root).canWrite()) throw new IllegalArgumentException("Unable to build the Changeset structures with a non-writeable content store root folder.");
this.root = root;
return this;
}
/**
* We are interested in building an empty, immutable {@link ChangeSet} instance.
*
* @return this builder, for being used in a fluent mode.
*/
Builder empty()
{
this.root = null;
this.immutable = true;
return this;
}
/**
* Builds the product of this builder (i.e. the {@link ChangeSet} instance).
*
* @return the product of this builder (i.e. the {@link ChangeSet} instance).
*/
ChangeSet build()
{
if (root == null)
{
// Creates a transient changeset (no persistence); mainly used for reducing the changes during replication.
return new ChangeSet(
immutable ? emptySet() : new HashSet<>(),
immutable ? emptySet() : new HashSet<>());
}
IndexWriter writer = null;
try
{
File indexDirectory = new File(root, CHANGESETS_ROOT_FOLDER_NAME);
writer = new IndexWriter(FSDirectory.open(indexDirectory.toPath()), new IndexWriterConfig());
writer.commit();
final SearcherManager searcher = new SearcherManager(writer, null);
LOGGER.info("ContentStore Changeset index has been correctly mounted on {}", indexDirectory.getAbsolutePath());
return new ChangeSet(
searcher,
writer,
immutable ? emptySet() : new HashSet<>(),
immutable ? emptySet() : new HashSet<>());
}
catch (Exception exception)
{
ofNullable(writer).ifPresent(ChangeSet::silentyClose);
throw new IllegalArgumentException("Unable to create a ContentStore ChangeSet data structure. See further details in the stacktrack below.", exception);
}
}
}
private final static String VERSION_FIELD_NAME = "version";
private final static String RVERSION_FIELD_NAME = "rversion";
private final static String ADDS_FIELD_NAME = "adds";
private final static String DELETES_FIELD_NAME = "deletes";
final static String CHANGESETS_ROOT_FOLDER_NAME = "changeSets";
Set<String> deletes;
Set<String> adds;
SearcherManager searcher;
private final IndexWriter writer;
Query selectEverything = new MatchAllDocsQuery();
/**
* Builds a new transient {@link ChangeSet} with the given Lucene facades.
*
* @param deletesContainer the container which will hold the deletes.
* @param addsContainer the container which will hold the adds/updates.
*/
private ChangeSet(
final Set<String> deletesContainer,
final Set<String> addsContainer)
{
this(null, null, deletesContainer, addsContainer);
}
/**
* Builds a new {@link ChangeSet} with the given Lucene facades.
*
* @param searcher the searcher reference (actually a {@link SearcherManager} instance instead of dealing with {@link IndexSearcher} directly.
* @param writer the {@link IndexWriter} instance used for persisting the content store changes.
* @param deletesContainer the container which will hold the deletes.
* @param addsContainer the container which will hold the adds/updates.
*/
private ChangeSet(
final SearcherManager searcher,
final IndexWriter writer,
final Set<String> deletesContainer,
final Set<String> addsContainer)
{
this.searcher = searcher;
this.writer = writer;
this.deletes = deletesContainer;
this.adds = addsContainer;
}
/**
* Records a delete change.
*
* @param path the relative path of the file which has been deleted.
*/
synchronized void delete(String path)
{
adds.remove(path);
deletes.add(path);
LOGGER.debug("ContentStore change recorded: item {} has been deleted.", path);
debugPendingChanges();
}
/**
* Records an add or update change.
*
* @param path the relative path of the file which has been updated or added.
*/
synchronized void addOrReplace(String path)
{
deletes.remove(path);
adds.add(path);
LOGGER.debug("ContentStore change recorded: item {} has been added/updated.", path);
debugPendingChanges();
}
/**
* Flushes all pending collected content store changes.
*
* @throws IOException in case of I/O failure.
*/
void flush() throws IOException
{
// No ops if this is a transient changeset
if (searcher == null || writer == null) {
return;
}
if (adds.isEmpty() && deletes.isEmpty())
{
LOGGER.debug("No changes in contentstore to flush");
return;
}
final long version = System.currentTimeMillis();
LOGGER.debug("About to add a new Changeset entry (version = {}, deletes = {}, adds = {})", version, deletes.size(), adds.size());
final Document document = new Document();
document.add(new NumericDocValuesField(VERSION_FIELD_NAME, version));
document.add(new LongPoint(RVERSION_FIELD_NAME, version));
Set<String> tmpDel;
Set<String> tmpAdd;
synchronized (this) {
tmpDel = deletes;
deletes = new HashSet<>();
tmpAdd = adds;
adds = new HashSet<>();
}
tmpDel.stream()
.map(item -> new StoredField(DELETES_FIELD_NAME, item))
.forEach(document::add);
tmpAdd.stream()
.map(item -> new StoredField(ADDS_FIELD_NAME, item))
.forEach(document::add);
writer.addDocument(document);
writer.commit();
searcher.maybeRefresh();
LOGGER.debug("New Changeset entry have been added (version = {}, deletes = {}, adds = {})", version, deletes.size(), adds.size());
}
/**
* Sanity check for making sure the requestor sent a valid and known content store version.
*
* @param version the content store version (sent by a requestor)
* @return true if the given version is unknown, that is, is not part of the content store version history.
*/
boolean isUnknownVersion(long version)
{
try
{
TopDocs hits = searcher().search(LongPoint.newExactQuery(RVERSION_FIELD_NAME, version), 1);
return hits.totalHits != 1;
}
catch(Exception exception)
{
LOGGER.error("Unable to check the requested version ({}) in the local versioning store. See further details in the stacktrace below.", version, exception);
return true;
}
}
@Override
public void close()
{
ofNullable(writer).ifPresent(ChangeSet::silentyClose);
ofNullable(searcher).ifPresent(ChangeSet::silentyClose);
}
/**
* Returns the last persisted content store version.
*
* @return the last persisted content store version, SolrContentStore#NO_VERSION_AVAILABLE in case the version isn't available.
*/
long getLastCommittedVersion()
{
try
{
TopDocs hits = searcher().search(
selectEverything,
1,
new Sort(new SortedNumericSortField(VERSION_FIELD_NAME, SortField.Type.LONG, true)),
false,
false);
return ofNullable(hits.scoreDocs)
.filter(docs -> docs.length > 0)
.map(docs -> ((FieldDoc) docs[0]).fields)
.filter(fields -> fields.length > 0)
.map(fields -> (Long)fields[0])
.orElse(SolrContentStore.NO_VERSION_AVAILABLE);
}
catch(Exception exception)
{
LOGGER.error("Unable to retrieve the last committed content store changeset version. " +
"As consequence of that a dummy value of " + SolrContentStore.NO_VERSION_AVAILABLE +
" will be returned. See further details in the stacktrack below.",
exception);
return SolrContentStore.NO_VERSION_AVAILABLE;
}
}
/**
* Returns the content store changes (adds / deletes) since the given version (exclusive).
*
* @param version the start offset version (exclusive).
* @return the content store changes (adds / deletes) since the given version (exclusive).
*/
public ChangeSet since(long version)
{
try
{
Query query = LongPoint.newRangeQuery(RVERSION_FIELD_NAME, Math.addExact(version, 1), Long.MAX_VALUE);
TopDocs hits = searcher().search(
query,
100,
new Sort(new SortedNumericSortField(VERSION_FIELD_NAME, SortField.Type.LONG)),
false,
false);
final BiFunction<ChangeSet, ? super Pair<List<String>,List<String>>, ChangeSet> accumulator =
(partial, nth) -> {
final List<String> nthDeletes = nth.getFirst();
final List<String> nthAdds = nth.getSecond();
nthDeletes.forEach(partial::delete);
nthAdds.forEach(partial::addOrReplace);
return partial;
};
final BinaryOperator<ChangeSet> combiner = (c1, c2) -> {
c1.deletes.forEach(c2::delete);
c1.adds.forEach(c2::addOrReplace);
return c1;
};
return stream(hits.scoreDocs)
.map(this::toDoc)
.map(doc ->
new Pair<>(
asList(doc.getValues(DELETES_FIELD_NAME)),
asList(doc.getValues(ADDS_FIELD_NAME))))
.reduce(new ChangeSet.Builder().build(), accumulator, combiner);
}
catch(Exception exception)
{
LOGGER.error("Unable to retrieve the changeset since version {}. " +
"As consequence of that an empty result will be returned. " +
"See further details in the stacktrack below.",
version,
exception);
return EMPTY_CHANGESET;
}
}
private Document toDoc(ScoreDoc hit)
{
try
{
return searcher().doc(hit.doc);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
private void debugPendingChanges()
{
if (LOGGER.isDebugEnabled())
{
LOGGER.debug("ContentStore pending deletes: " + deletes);
LOGGER.debug("ContentStore pending adds/updates: " + adds);
}
}
/**
* Silently close (i.e. without any exception re-throwing) the incoming resource.
*
* @param resource the closeable resource.
*/
private static void silentyClose(Closeable resource)
{
try
{
resource.close();
}
catch (Exception exception)
{
LOGGER.error("Unable to properly close the resource instance {}. See further details in the stacktrace below.", resource, exception);
}
}
private IndexSearcher searcher() throws IOException
{
return searcher.acquire();
}
}
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
/**
* {@link AccessMode} specialisation for those modes that need some kind of initialisation.
*
* @author Andrea Gazzarini
* @since 1.5
*/
public interface InitialisableAccessMode extends AccessMode
{
/**
* Initialises this access mode instance.
*/
void init();
}
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
@@ -18,52 +18,475 @@
*/
package org.alfresco.solr.content;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.alfresco.repo.content.ContentContext;
import org.alfresco.repo.content.ContentStore;
import org.alfresco.service.cmr.repository.ContentReader;
import org.alfresco.service.cmr.repository.ContentWriter;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.client.NodeMetaData;
import org.alfresco.solr.config.ConfigUtil;
import org.alfresco.solr.handler.AlfrescoReplicationHandler;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.SnapShooter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Predicate;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.Optional.of;
import static java.util.Optional.ofNullable;
import static java.util.stream.Collectors.toList;
import static org.alfresco.solr.content.SolrContentUrlBuilder.FILE_EXTENSION;
import static org.alfresco.solr.content.SolrContentUrlBuilder.logger;
/**
* A content store specific to SOLR's requirements: The URL is generated from a
* set of properties such as:
*
* <ul>
* <li>ACL ID</li>
* <li>DB ID</li>
* <li>Other metadata</li>
* <li>ACL ID</li>
* <li>DB ID</li>
* <li>Other metadata</li>
* </ul>
* The URL, if not known, can be reliably regenerated using the
* {@link SolrContentUrlBuilder}.
*
*
* The URL, if not known, can be reliably regenerated using the {@link SolrContentUrlBuilder}.
* <br/>
*
* Since version 1.5 this class acts as a logical singleton: there will be only one instance per node.
* That reflects exactly what we physically have in the filesystem (i.e. there's only one content store per node).
*
* That unique instance is created at startup in {@link org.alfresco.solr.AlfrescoCoreAdminHandler} and then passed to
* each registered core (see {@link org.alfresco.solr.lifecycle.SolrCoreLoadListener})
*
* The State pattern implemented by means of the {@link AccessMode} interface allows for a given {@link SolrContentStore} instance to act:
*
* <ul>
* <li>in READ/WRITE mode: when <b>at least one core</b> of the hosting node is a master or it is a standalone shard/instance.</li>
* <li>in READ ONLY mode: when <b>all cores</b> of the hosting node are slaves.</li>
* </ul>
*
* The Finite State Machine (FST) provides three possible states: Initial, Read Only, Read/Write.
* The allowed transitions are:
*
* <ul>
* <li>Initial -> ReadOnly: a slave core has been registered, the content store hasn't been yet initialised.</li>
* <li>
* ReadOnly -> Read/Write: this scenario is unusual because we should have on the same instance a mixed set of cores
* (some master/standalone, some slaves). It happens when a first slave core registers and causes the transition
* described in the first point (initial -> read only). Then a master or standalone core registers so we need to
* move from a read only to a complete readable/writable managed content store.
* </li>
* <li> Initial -> Read/Write: a master or standalone core has been registered, and the content store hasn't been yet initialised.</li>
* </ul>
*
* Note the following transitions are not allowed:
*
* <ul>
* <li>Coming back to Initial state: once it has been initialised the Content Store cannot return back to the "Initial" state.</li>
* <li>
* Read/Write -> ReadOnly: if at least one master or standalone core registers, the content store is permanently moved in Read/Write mode.
* So even a further slave node registers (not very usually as described above) the content store remains in RW mode.
* </li>
* </ul>
*
* @author Derek Hulley
* @author Michael Suzuki
* @since 5.0
* @author Andrea Gazzarini
* @since 1.5
* @see org.alfresco.solr.lifecycle.SolrCoreLoadListener
* @see <a href="https://it.wikipedia.org/wiki/State_pattern">State Pattern</a>
*/
public class SolrContentStore implements ContentStore
public final class SolrContentStore implements Closeable, AccessMode
{
protected final static Logger log = LoggerFactory.getLogger(SolrContentStore.class);
private final static Logger LOGGER = LoggerFactory.getLogger(SolrContentStore.class);
public static final long NO_VERSION_AVAILABLE = -1L;
public static final long NO_CONTENT_STORE_REPLICATION_REQUIRED = -2L;
static final String CONTENT_STORE = "contentstore";
static final String SOLR_CONTENT_DIR = "solr.content.dir";
public static final String INFO = "info";
public static final String FULL_REPLICATION = "full-replication";
public static final String DELETES = "deletes";
public static final String ADDS = "adds";
private final Predicate<File> onlyDatafiles = file -> file.isFile() && file.getName().endsWith(FILE_EXTENSION);
private final String root;
/**
* Constructor.
* @param solrHome
* Used for denoting the very beginning state, when the {@link SolrContentStore} has been created
* but we don't know (yet) which is the role played by the cores that will register on the hosting Solr node.
*/
AccessMode notYetSet = new AccessMode()
{
@Override
public long getLastCommittedVersion()
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void setLastCommittedVersion(long version)
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public Map<String, List<Map<String, Object>>> getChanges(long version)
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc)
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc)
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void removeDocFromContentStore(NodeMetaData nodeMetaData)
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void flushChangeSet()
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void switchOnReadOnlyMode()
{
logger.info("Switching the content store to ReadOnly mode.");
currentAccessMode = readOnly;
}
@Override
public void switchOnReadWriteMode()
{
logger.info("Switching the content store to Read/Write mode.");
readWrite.init();
currentAccessMode = readWrite;
}
@Override
public void close()
{
// Nothing to close here
}
};
final InitialisableAccessMode readOnly = new InitialisableAccessMode()
{
@Override
public void init()
{
// Nothing to be done here...
}
@Override
public void switchOnReadWriteMode()
{
logger.info("Switching from ReadOnly to Read/Write Content Store.");
readWrite.init();
currentAccessMode = readWrite;
logger.info("Switching from ReadOnly to Read/Write Content Store.");
}
@Override
public void switchOnReadOnlyMode()
{
logger.info("The content store is already in ReadOnly mode so this call won't have any effect.");
}
@Override
public long getLastCommittedVersion()
{
try
{
return Files.lines(Paths.get(root, ".version"))
.map(Long::parseLong)
.findFirst()
.orElse(NO_VERSION_AVAILABLE);
}
catch (Exception e)
{
return NO_VERSION_AVAILABLE;
}
}
@Override
public void setLastCommittedVersion(long version)
{
try
{
File tmpFile = new File(root, ".version-" + new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date()));
FileWriter wr = new FileWriter(tmpFile);
wr.write(Long.toString(version));
wr.close();
tmpFile.renameTo(new File(root, ".version"));
}
catch (IOException exception)
{
logger.error("Unable to persist the last committed content store version {}. See the stacktrace below for furtger details.", version, exception);
}
}
@Override
public Map<String, List<Map<String, Object>>> getChanges(long version)
{
logger.warn("NoOp SolrContentStore changes call on slave side: this shouldn't happen because the ContentStore is in read-only mode when the hosting node is a slave.");
return emptyMap();
}
@Override
public void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc)
{
logger.warn("NoOp SolrContentStore write call on slave side: this shouldn't happen because the ContentStore is in read-only mode when the hosting node is a slave.");
}
@Override
public void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc)
{
logger.warn("NoOp SolrContentStore write call on slave side: this shouldn't happen because the ContentStore is in read-only mode when the hosting node is a slave.");
}
@Override
public void removeDocFromContentStore(NodeMetaData nodeMetaData)
{
logger.warn("NoOp SolrContentStore write call on slave side: this shouldn't happen because the ContentStore is in read-only mode when the hosting node is a slave.");
}
@Override
public void flushChangeSet()
{
logger.warn("NoOp ChangeSet tracking call on slave side: this shouldn't happen because the ContentStore is in read-only mode when the hosting node is a slave.");
}
@Override
public void close()
{
// There's nothing to close on slave side
}
};
final InitialisableAccessMode readWrite = new InitialisableAccessMode()
{
private ChangeSet changeSet;
@Override
public void init()
{
changeSet = ofNullable(changeSet).orElseGet(() -> new ChangeSet.Builder().withContentStoreRoot(root).build());
}
@Override
public long getLastCommittedVersion()
{
return changeSet.getLastCommittedVersion();
}
@Override
public void setLastCommittedVersion(long version)
{
// Do nothing here, as we are on master-side
}
@Override
public Map<String, List<Map<String, Object>>> getChanges(long version)
{
// The slave doesn't have a version, we are listing the whole content store
if (version <= NO_VERSION_AVAILABLE || changeSet.isUnknownVersion(version))
{
String message = "A slave requested the content store synchronization " +
((version <= NO_VERSION_AVAILABLE)
? " without providing any local version (actually {})."
: " with an invalid/unknown local version number ({}).") +
"As consequence of that this master will list the whole content store because a full replication is needed.";
logger.info(message, version);
return Map.of(
INFO, singletonList(Map.of(FULL_REPLICATION, true)),
ADDS, fullContentStore(),
DELETES, emptyList());
}
ChangeSet changes = changeSet.since(version);
return Map.of(
INFO, singletonList(Map.of(FULL_REPLICATION, false)),
DELETES,
changes.deletes.stream()
.map(path -> Map.<String, Object>of("name", path))
.collect(toList()),
ADDS,
changes.adds.stream()
.map(relativePath -> root + relativePath)
.map(File::new)
.map(file -> new AlfrescoReplicationHandler.FileInfo(file, file.getAbsolutePath().replace(root, "")))
.map(AlfrescoReplicationHandler.FileInfo::getAsMap)
.collect(toList()));
}
@Override
public void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc)
{
ContentContext contentContext =
of(SolrContentUrlBuilder
.start()
.add(SolrContentUrlBuilder.KEY_TENANT, tenant)
.add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(dbId)))
.map(SolrContentUrlBuilder::getContentContext)
.orElseThrow(() -> new IllegalArgumentException("Unable to build a Content Context from tenant " + tenant + " and DBID " + dbId));
this.delete(contentContext.getContentUrl());
ContentWriter writer = this.getWriter(contentContext);
LOGGER.debug("Writing {}/{} to {}", tenant, dbId, contentContext.getContentUrl());
try (OutputStream contentOutputStream = writer.getContentOutputStream();
GZIPOutputStream gzip = new GZIPOutputStream(contentOutputStream))
{
JavaBinCodec codec = new JavaBinCodec(resolver);
codec.marshal(doc, gzip);
File file = getFileFromUrl(contentContext.getContentUrl());
changeSet.addOrReplace(relativePath(file));
}
catch (Exception exception)
{
LOGGER.warn("Unable to write to Content Store using URL: {}", contentContext.getContentUrl(), exception);
}
}
@Override
public void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc) {
String fixedTenantDomain = AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain());
storeDocOnSolrContentStore(fixedTenantDomain, nodeMetaData.getId(), doc);
}
@Override
public void removeDocFromContentStore(NodeMetaData nodeMetaData)
{
String fixedTenantDomain = AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain());
String contentUrl = SolrContentUrlBuilder
.start()
.add(SolrContentUrlBuilder.KEY_TENANT, fixedTenantDomain)
.add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(nodeMetaData.getId()))
.getContentContext()
.getContentUrl();
delete(contentUrl);
}
@Override
public void flushChangeSet() throws IOException
{
changeSet.flush();
}
@Override
public void switchOnReadWriteMode()
{
logger.debug("The content store is already in ReadWrite mode; as consequence of that, the incoming \"SET-TO-RW-MODE\" call won't have any effect.");
}
@Override
public void switchOnReadOnlyMode()
{
logger.debug("A writable content store cannot switch in ReadOnly mode. This could happen in an edge case where" +
" on the same Solr node we have masters and slaves nodes");
}
@Override
public void close()
{
changeSet.close();
}
private List<Map<String, Object>> fullContentStore()
{
try
{
return Files.walk(Paths.get(root))
.map(Path::toFile)
.filter(onlyDatafiles)
.map(file -> new AlfrescoReplicationHandler.FileInfo(file, file.getAbsolutePath().replace(root, "")))
.map(AlfrescoReplicationHandler.FileInfo::getAsMap)
.collect(toList());
}
catch (Exception e)
{
LOGGER.error("An exception occurred while retrieving the whole ContentStore filelist. " +
"As consequence of that an empty list will be returned (i.e. no ContentStore synch will happen).");
return emptyList();
}
}
private void delete(String contentUrl)
{
File file = getFileFromUrl(contentUrl);
if (file.delete()) changeSet.delete(relativePath(file));
}
private ContentWriter getWriter(ContentContext context)
{
String url = context.getContentUrl();
File file = getFileFromUrl(url);
return new SolrFileContentWriter(file, url);
}
};
AccessMode currentAccessMode = notYetSet;
private final JavaBinCodec.ObjectResolver resolver = (o, codec) -> {
if(o instanceof BytesRef)
{
BytesRef br = (BytesRef)o;
codec.writeByteArray(br.bytes,br.offset,br.length);
return null;
}
return o;
};
/**
* Builds a new {@link SolrContentStore} instance with the given SOLR HOME.
*
* @param solrHome the Solr HOME.
*/
public SolrContentStore(String solrHome)
{
@@ -77,122 +500,92 @@ public class SolrContentStore implements ContentStore
{
//Its very unlikely that solrHome would not exist so we will log an error
//but continue because solr.content.dir may be specified, so it keeps working
log.error(solrHomeFile.getAbsolutePath() + " does not exist.");
LOGGER.error(solrHomeFile.getAbsolutePath() + " does not exist.");
}
String path = solrHomeFile.getParent()+"/"+CONTENT_STORE;
log.warn(path + " will be used as a default path if " + SOLR_CONTENT_DIR + " property is not defined");
String path = solrHomeFile.getParent() + "/" + CONTENT_STORE;
LOGGER.warn(path + " will be used as a default path if " + SOLR_CONTENT_DIR + " property is not defined");
File rootFile = new File(ConfigUtil.locateProperty(SOLR_CONTENT_DIR, path));
try
{
FileUtils.forceMkdir(rootFile);
}
catch (Exception e)
{
}
catch (Exception e) {
throw new RuntimeException("Failed to create directory for content store: " + rootFile, e);
}
this.root = rootFile.getAbsolutePath();
}
// write a BytesRef as a byte array
private JavaBinCodec.ObjectResolver resolver = new JavaBinCodec.ObjectResolver()
/**
* Returns the content store changes since the given (requestor) version.
*
* @param version the requestor version.
* @return the content store changes since the given (requestor) version.
*/
@Override
public Map<String, List<Map<String, Object>>> getChanges(long version)
{
@Override public Object resolve(Object o,JavaBinCodec codec)throws IOException
{
if(o instanceof BytesRef)
{
BytesRef br=(BytesRef)o;
codec.writeByteArray(br.bytes,br.offset,br.length);
return null;
}
return o;
}
};
return currentAccessMode.getChanges(version);
}
/**
* Retrieve document from SolrContentStore.
*
* @param tenant identifier
* @param dbId identifier
* @return {@link SolrInputDocument} searched document
* @throws IOException if error
*/
public SolrInputDocument retrieveDocFromSolrContentStore(String tenant, long dbId) throws IOException
public SolrInputDocument retrieveDocFromSolrContentStore(String tenant, long dbId)
{
String contentUrl = SolrContentUrlBuilder.start().add(SolrContentUrlBuilder.KEY_TENANT, tenant)
.add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(dbId)).get();
String contentUrl =
SolrContentUrlBuilder.start()
.add(SolrContentUrlBuilder.KEY_TENANT, tenant)
.add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(dbId))
.get();
ContentReader reader = this.getReader(contentUrl);
SolrInputDocument cachedDoc = null;
if (reader.exists())
if (!reader.exists())
{
// try-with-resources statement closes all these InputStreams
try (InputStream contentInputStream = reader.getContentInputStream();
// Uncompresses the document
GZIPInputStream gzip = new GZIPInputStream(contentInputStream);)
{
cachedDoc = (SolrInputDocument) new JavaBinCodec(resolver).unmarshal(gzip);
} catch (Exception e)
{
// Don't fail for this
log.warn("Failed to get doc from store using URL: " + contentUrl, e);
return null;
}
return null;
}
try (InputStream contentInputStream = reader.getContentInputStream();
InputStream gzip = new GZIPInputStream(contentInputStream))
{
return (SolrInputDocument) new JavaBinCodec(resolver).unmarshal(gzip);
}
catch (Exception exception)
{
// Don't fail for this
LOGGER.warn("Failed to get doc from store using URL: " + contentUrl, exception);
return null;
}
return cachedDoc;
}
private final String root;
@Override
public boolean isContentUrlSupported(String contentUrl)
public long getLastCommittedVersion()
{
return (contentUrl != null && contentUrl.startsWith(SolrContentUrlBuilder.SOLR_PROTOCOL_PREFIX));
return currentAccessMode.getLastCommittedVersion();
}
/**
* @return <tt>true</tt> always
*/
@Override
public boolean isWriteSupported()
public void setLastCommittedVersion(long version)
{
return true;
currentAccessMode.setLastCommittedVersion(version);
}
/**
* @return -1 always
* Returns the absolute path of the content store root folder.
*
* @return the absolute path of the content store root folder.
*/
@Override
public long getSpaceFree()
{
return -1L;
}
/**
* @return -1 always
*/
@Override
public long getSpaceTotal()
{
return -1L;
}
@Override
public String getRootLocation()
{
return root;
}
/**
* Convert a content URL into a File, whether it exists or not
*/
private File getFileFromUrl(String contentUrl)
{
String path = contentUrl.replace(SolrContentUrlBuilder.SOLR_PROTOCOL_PREFIX, root + "/");
return new File(path);
}
@Override
public boolean exists(String contentUrl)
{
File file = getFileFromUrl(contentUrl);
@@ -200,93 +593,101 @@ public class SolrContentStore implements ContentStore
}
@Override
public ContentReader getReader(String contentUrl)
public void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc)
{
currentAccessMode.storeDocOnSolrContentStore(tenant, dbId, doc);
}
/**
* Store {@link SolrInputDocument} in to Alfresco solr content store.
*
* @param nodeMetaData the incoming node metadata.
* @param doc the document itself.
*/
@Override
public void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc)
{
currentAccessMode.storeDocOnSolrContentStore(nodeMetaData, doc);
}
/**
* Removes {@link SolrInputDocument} from Alfresco solr content store.
*
* @param nodeMetaData the incoming node metadata.
*/
@Override
public void removeDocFromContentStore(NodeMetaData nodeMetaData)
{
currentAccessMode.removeDocFromContentStore(nodeMetaData);
}
@Override
public void flushChangeSet() throws IOException
{
currentAccessMode.flushChangeSet();
}
@Override
public void close() throws IOException
{
currentAccessMode.close();
}
@Override
public void switchOnReadWriteMode()
{
currentAccessMode = readWrite;
}
@Override
public void switchOnReadOnlyMode()
{
currentAccessMode = readOnly;
}
/**
* Assuming the input file belongs to the content store, it returns the corresponding relative path.
*
* @param file the content store file.
* @return the relative file path.
*/
private String relativePath(File file)
{
return file.getAbsolutePath().replace(root, "");
}
/**
* Convert a content URL into a File, whether it exists or not
*/
private File getFileFromUrl(String contentUrl)
{
return new File(contentUrl.replace(SolrContentUrlBuilder.SOLR_PROTOCOL_PREFIX, root + "/"));
}
private ContentReader getReader(String contentUrl)
{
File file = getFileFromUrl(contentUrl);
return new SolrFileContentReader(file, contentUrl);
}
@Override
public ContentWriter getWriter(ContentContext context)
{
// Ensure that there is a context and that it has a URL
if (context == null || context.getContentUrl() == null)
{
throw new IllegalArgumentException("Retrieve a writer with a URL-providing ContentContext.");
}
String url = context.getContentUrl();
File file = getFileFromUrl(url);
SolrFileContentWriter writer = new SolrFileContentWriter(file, url);
// Done
return writer;
}
@Override
public boolean delete(String contentUrl)
{
File file = getFileFromUrl(contentUrl);
return file.delete();
}
/**
* Stores a {@link SolrInputDocument} into Alfresco solr content store.
* @param tenant
* @param dbId
* @param doc
* @throws IOException
* Enables/disables the content store access mode.
* The term "toggles" is just for indicating that this method will be called several times (one for each registered
* core). The underlying FSM makes sure this call will be idempotent so in case of read/write content store the
* data structure used for maintaining the content store versioning will be initialised only once, even if this
* method is called repeatedly.
*
* @param enableReadOnlyMode a flag indicating if the requesting core requires a readOnly (true) or readWrite (false) content store.
*/
public void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc) throws IOException
public synchronized void toggleReadOnlyMode(boolean enableReadOnlyMode)
{
ContentContext contentContext = SolrContentUrlBuilder
.start()
.add(SolrContentUrlBuilder.KEY_TENANT, tenant)
.add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(dbId))
.getContentContext();
this.delete(contentContext.getContentUrl());
ContentWriter writer = this.getWriter(contentContext);
if (log.isDebugEnabled())
if (enableReadOnlyMode)
{
log.debug("Writing doc to " + contentContext.getContentUrl());
currentAccessMode.switchOnReadOnlyMode();
}
try (
OutputStream contentOutputStream = writer.getContentOutputStream();
// Compresses the document
GZIPOutputStream gzip = new GZIPOutputStream(contentOutputStream);
)
else
{
JavaBinCodec codec = new JavaBinCodec(resolver);
codec.marshal(doc, gzip);
}
catch (Exception e)
{
// A failure to write to the store is acceptable as long as it's logged
log.warn("Failed to write to store using URL: " + contentContext.getContentUrl(), e);
currentAccessMode.switchOnReadWriteMode();
}
}
/**
* Store {@link SolrInputDocument} in to Alfresco solr content store.
* @param nodeMetaData identifier
* @param doc to store
* @throws IOException if error
*/
public void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc) throws IOException
{
String fixedTenantDomain = AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain());
storeDocOnSolrContentStore(fixedTenantDomain, nodeMetaData.getId(), doc);
}
/**
* Removes {@link SolrInputDocument} from Alfresco solr content store.
* @param nodeMetaData
*/
public void removeDocFromContentStore(NodeMetaData nodeMetaData)
{
String fixedTenantDomain = AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain());
String contentUrl = SolrContentUrlBuilder
.start()
.add(SolrContentUrlBuilder.KEY_TENANT, fixedTenantDomain)
.add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(nodeMetaData.getId()))
.getContentContext()
.getContentUrl();
this.delete(contentUrl);
}
}
}
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
@@ -18,7 +18,7 @@
*/
package org.alfresco.solr.content;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.TreeMap;
import java.util.zip.CRC32;
@@ -43,20 +43,13 @@ import org.slf4j.LoggerFactory;
*/
public class SolrContentUrlBuilder
{
/**
* <b>solr</b> is the prefix for SOLR content URLs
* @see #isContentUrlSupported(String)
*/
public static final String SOLR_PROTOCOL = "solr";
public static final String SOLR_PROTOCOL_PREFIX = SOLR_PROTOCOL + ContentStore.PROTOCOL_DELIMITER;
public static final String FILE_EXTENSION = ".gz";
private static final String SOLR_PROTOCOL = "solr";
static final String SOLR_PROTOCOL_PREFIX = SOLR_PROTOCOL + ContentStore.PROTOCOL_DELIMITER;
static final String FILE_EXTENSION = ".gz";
/** The key for the tenant name */
public static final String KEY_TENANT = "tenant";
/** The key for the DB ID */
public static final String KEY_DB_ID = "dbId";
/** The key for the ACL ID */
public static final String KEY_ACL_ID = "aclId";
static final String KEY_TENANT = "tenant";
static final String KEY_DB_ID = "dbId";
static final String KEY_ACL_ID = "aclId";
protected final static Logger logger = LoggerFactory.getLogger(SolrContentUrlBuilder.class);
@@ -66,9 +59,9 @@ public class SolrContentUrlBuilder
/**
* Protected constructor used by {@link SolrContentUrlBuilder#start()}
*/
protected SolrContentUrlBuilder()
private SolrContentUrlBuilder()
{
this.metadata = new TreeMap<String, String>();
this.metadata = new TreeMap<>();
}
/**
@@ -83,32 +76,34 @@ public class SolrContentUrlBuilder
/**
* Add some metadata to the URL generator. The order in which metadata is added is irrelevant.
* <p/>
* Note that there are specific keys that are commonly used and, if provided, may not be null or empty.
*
* <ul>
* <li><b>{@link #KEY_TENANT}:</b> The name of the tenant or 'default' if missing.</li>
* <li><b>{@link #KEY_DB_ID}:</b> The database ID.</li>
* <li><b>{@link #KEY_ACL_ID}:</b> The ACL ID.</li>
* </ul>
*
* @param key an arbitrary metadata key (never <tt>null</tt>>
* @param value some metadata value (<tt>null</tt> is supported)
* @return this builder for more building
* @param key an arbitrary metadata key (never <tt>null</tt>>
* @param value some metadata value (<tt>null</tt> is supported)
* @return this builder for more building
*
* @throws IllegalArgumentException if the key is null
* @throws IllegalStateException if the key has been used already
* @throws IllegalArgumentException if the key is null
* @throws IllegalStateException if the key has been used already
*/
public synchronized SolrContentUrlBuilder add(String key, String value)
public SolrContentUrlBuilder add(String key, String value)
{
if (key == null)
{
throw new IllegalArgumentException("The metadata 'key' may not be null.");
}
String previous = metadata.put(key, value);
if (previous != null)
{
throw new IllegalStateException("The metadata key, '" + key + "', has already been used.");
}
// Check well-known keys
if (key.equals(KEY_TENANT) || key.equals(KEY_DB_ID) || key.equals(KEY_ACL_ID))
{
@@ -131,8 +126,8 @@ public class SolrContentUrlBuilder
/**
* Get the final content URL using the {@link #add(String, String) supplied metadata}.
*
* @return the SOLR content URL
* @throws IllegalStateException if no metadata has been added
* @return the SOLR content URL
* @throws IllegalStateException if no metadata has been added
*/
public synchronized String get()
{
@@ -169,21 +164,13 @@ public class SolrContentUrlBuilder
sb.append("misc/");
// Calculate the CRC
CRC32 crc = new CRC32();
try
for (Map.Entry<String, String> entry : metadata.entrySet())
{
for (Map.Entry<String, String> entry : metadata.entrySet())
{
// This is ordered, so just add each entry as "key = value".
// DO NOT USE entry.toString() because the format is not a contract
// and we have to have the same string for the same metadata
String entryStr = entry.getKey() + "=" + entry.getValue() + "; ";
crc.update(entryStr.getBytes("UTF-8"));
}
}
catch (UnsupportedEncodingException e)
{
// Yeah, right.
throw new RuntimeException("UTF-8 is not supported.", e);
// This is ordered, so just add each entry as "key = value".
// DO NOT USE entry.toString() because the format is not a contract
// and we have to have the same string for the same metadata
String entryStr = entry.getKey() + "=" + entry.getValue() + "; ";
crc.update(entryStr.getBytes(StandardCharsets.UTF_8));
}
numSb.append(crc.getValue());
}
@@ -218,7 +205,7 @@ public class SolrContentUrlBuilder
/**
* Helper method to retrieve a {@link ContentContext} constructed using the final {@link #get()} url.
*/
public ContentContext getContentContext()
ContentContext getContentContext()
{
String url = get();
return new ContentContext(null, url);
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
@@ -42,7 +42,7 @@ import org.springframework.util.FileCopyUtils;
* @author Derek Hulley
* @since 5.0
*/
public class SolrFileContentReader implements ContentReader
class SolrFileContentReader implements ContentReader
{
private final File file;
private final String contentUrl;
@@ -51,7 +51,7 @@ public class SolrFileContentReader implements ContentReader
* @param file the file to write to
* @param contentUrl the content URL for information purposes
*/
protected SolrFileContentReader(File file, String contentUrl)
SolrFileContentReader(File file, String contentUrl)
{
this.file = file;
this.contentUrl = contentUrl;
@@ -120,9 +120,8 @@ public class SolrFileContentReader implements ContentReader
}
try
{
InputStream is = new BufferedInputStream(new FileInputStream(file));
// done
return is;
return new BufferedInputStream(new FileInputStream(file));
}
catch (Throwable e)
{
@@ -192,12 +191,9 @@ public class SolrFileContentReader implements ContentReader
ByteArrayOutputStream os = new ByteArrayOutputStream();
FileCopyUtils.copy(is, os); // both streams are closed
byte[] bytes = os.toByteArray();
// get the encoding for the string
String encoding = "UTF-8";
// create the string from the byte[] using encoding if necessary
String content = (encoding == null) ? new String(bytes) : new String(bytes, encoding);
// done
return content;
return new String(bytes, encoding);
}
catch (IOException e)
{
@@ -266,4 +262,4 @@ public class SolrFileContentReader implements ContentReader
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
}
}
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
@@ -41,7 +41,7 @@ import org.apache.commons.io.FileUtils;
* @author Derek Hulley
* @since 5.0
*/
public class SolrFileContentWriter implements ContentWriter
class SolrFileContentWriter implements ContentWriter
{
private final File file;
private final String contentUrl;
@@ -51,11 +51,10 @@ public class SolrFileContentWriter implements ContentWriter
* @param file the file to write to
* @param contentUrl the content URL for information purposes
*/
protected SolrFileContentWriter(File file, String contentUrl)
SolrFileContentWriter(File file, String contentUrl)
{
this.file = file;
this.contentUrl = contentUrl;
this.written = false;
}
@Override
@@ -109,7 +108,7 @@ public class SolrFileContentWriter implements ContentWriter
@Override
public synchronized OutputStream getContentOutputStream() throws ContentIOException
{
if (written == true)
if (written)
{
throw new IllegalStateException("The writer has already been used: " + file);
}
@@ -117,6 +116,7 @@ public class SolrFileContentWriter implements ContentWriter
{
throw new IllegalStateException("The file already exists: " + file);
}
try
{
OutputStream is = new BufferedOutputStream(FileUtils.openOutputStream(file));
@@ -139,7 +139,7 @@ public class SolrFileContentWriter implements ContentWriter
@Override
public synchronized void putContent(InputStream is) throws ContentIOException
{
if (written == true)
if (written)
{
throw new IllegalStateException("The writer has already been used: " + file);
}
@@ -147,6 +147,7 @@ public class SolrFileContentWriter implements ContentWriter
{
throw new IllegalStateException("The file already exists: " + file);
}
try
{
FileUtils.copyInputStreamToFile(is, file);
@@ -161,7 +162,7 @@ public class SolrFileContentWriter implements ContentWriter
@Override
public synchronized void putContent(File sourceFile) throws ContentIOException
{
if (written == true)
if (written)
{
throw new IllegalStateException("The writer has already been used: " + this.file);
}
@@ -173,6 +174,7 @@ public class SolrFileContentWriter implements ContentWriter
{
throw new IllegalStateException("The source file does not exist: " + sourceFile);
}
try
{
FileUtils.copyFile(sourceFile, this.file, false);
@@ -0,0 +1,24 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* The package contains all components that manage the Solr ContentStore.
* Note that in SearchServices 2.0 the whole package will be deprecated/removed because the content store will be
* replaced by the built-in Solr storage capability (i.e. stored fields).
*/
package org.alfresco.solr.content;
@@ -0,0 +1,102 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*
* Modification copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>
*
*/
package org.alfresco.solr.handler;
import org.apache.solr.handler.SnapShooter;
import java.net.URI;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
class OldBackupDirectory implements Comparable<OldBackupDirectory>
{
private static final Pattern dirNamePattern = Pattern.compile("^snapshot[.](.*)$");
private URI basePath;
private String dirName;
private Optional<Date> timestamp = Optional.empty();
OldBackupDirectory(URI basePath, String dirName)
{
this.dirName = Objects.requireNonNull(dirName);
this.basePath = Objects.requireNonNull(basePath);
Matcher m = dirNamePattern.matcher(dirName);
if (m.find())
{
try
{
this.timestamp = Optional.of(new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).parse(m.group(1)));
}
catch (ParseException e)
{
this.timestamp = Optional.empty();
}
}
}
public URI getPath()
{
return this.basePath.resolve(dirName);
}
String getDirName()
{
return dirName;
}
public Optional<Date> getTimestamp()
{
return timestamp;
}
@Override
public int compareTo(OldBackupDirectory that)
{
if (this.timestamp.isPresent() && that.timestamp.isPresent())
{
return that.timestamp.get().compareTo(this.timestamp.get());
}
// Use absolute value of path in case the time-stamp is missing on either side.
return that.getPath().compareTo(this.getPath());
}
}
@@ -0,0 +1,377 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*
* Modification copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>
*
*/
package org.alfresco.solr.handler;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.backup.repository.BackupRepository.PathType;
import org.apache.solr.core.backup.repository.LocalFileSystemRepository;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Consumer;
/**
* <p> Provides functionality equivalent to the snapshooter script </p>
* This is no longer used in standard replication.
*
* @since solr 1.4
*/
public class SnapShooter
{
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private SolrCore solrCore;
private String snapshotName = null;
private String directoryName = null;
private URI baseSnapDirPath = null;
private URI snapshotDirPath = null;
private BackupRepository backupRepo = null;
private String commitName; // can be null
@Deprecated
public SnapShooter(SolrCore core, String location, String snapshotName)
{
String snapDirStr;
// Note - This logic is only applicable to the usecase where a shared file-system is exposed via
// local file-system interface (primarily for backwards compatibility). For other use-cases, users
// will be required to specify "location" where the backup should be stored.
if (location == null)
{
snapDirStr = core.getDataDir();
}
else
{
snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString();
}
initialize(new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null);
}
SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName)
{
initialize(backupRepo, core, location, snapshotName, commitName);
}
private void initialize(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName)
{
this.solrCore = Objects.requireNonNull(core);
this.backupRepo = Objects.requireNonNull(backupRepo);
this.baseSnapDirPath = location;
this.snapshotName = snapshotName;
if (snapshotName != null)
{
directoryName = "snapshot." + snapshotName;
}
else
{
SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.ROOT);
directoryName = "snapshot." + fmt.format(new Date());
}
this.snapshotDirPath = backupRepo.resolve(location, directoryName);
this.commitName = commitName;
}
public BackupRepository getBackupRepository()
{
return backupRepo;
}
/**
* Gets the parent directory of the snapshots. This is the {@code location}
* given in the constructor.
*/
public URI getLocation()
{
return this.baseSnapDirPath;
}
void validateDeleteSnapshot()
{
Objects.requireNonNull(this.snapshotName);
boolean dirFound = false;
String[] paths;
try
{
paths = backupRepo.listAll(baseSnapDirPath);
for (String path : paths)
{
if (path.equals(this.directoryName)
&& backupRepo.getPathType(baseSnapDirPath.resolve(path)) == PathType.DIRECTORY)
{
dirFound = true;
break;
}
}
if (!dirFound)
{
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Snapshot " + snapshotName + " cannot be found in directory: " + baseSnapDirPath);
}
}
catch (IOException e)
{
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unable to find snapshot " + snapshotName + " in directory: " + baseSnapDirPath, e);
}
}
void deleteSnapAsync(final AlfrescoReplicationHandler alfrescoReplicationHandler)
{
new Thread(() -> deleteNamedSnapshot(alfrescoReplicationHandler)).start();
}
void validateCreateSnapshot() throws IOException
{
// Note - Removed the current behavior of creating the directory hierarchy.
// Do we really need to provide this support?
if (!backupRepo.exists(baseSnapDirPath))
{
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
" Directory does not exist: " + snapshotDirPath);
}
if (backupRepo.exists(snapshotDirPath))
{
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Snapshot directory already exists: " + snapshotDirPath);
}
}
public NamedList createSnapshot() throws Exception
{
RefCounted<SolrIndexSearcher> searcher = solrCore.getSearcher();
try
{
if (commitName != null)
{
SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager();
Optional<IndexCommit> commit = snapshotMgr.getIndexCommitByName(commitName);
if (commit.isPresent())
{
return createSnapshot(commit.get());
}
throw new SolrException(ErrorCode.SERVER_ERROR,
"Unable to find an index commit with name " + commitName + " for core " + solrCore.getName());
}
else
{
//TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first?
IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy();
IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit();
deletionPolicy.saveCommitPoint(indexCommit.getGeneration());
try
{
return createSnapshot(indexCommit);
}
finally
{
deletionPolicy.releaseCommitPoint(indexCommit.getGeneration());
}
}
}
finally
{
searcher.decref();
}
}
void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, Consumer<NamedList> result)
{
solrCore.getDeletionPolicy().saveCommitPoint(indexCommit.getGeneration());
new Thread(() -> {
try
{
result.accept(createSnapshot(indexCommit));
}
catch (Exception e)
{
LOG.error("Exception while creating snapshot", e);
NamedList snapShootDetails = new NamedList<>();
snapShootDetails.add("snapShootException", e.getMessage());
result.accept(snapShootDetails);
}
finally
{
solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration());
}
if (snapshotName == null)
{
try
{
deleteOldBackups(numberToKeep);
}
catch (IOException e)
{
LOG.warn("Unable to delete old snapshots ", e);
}
}
}).start();
}
// note: remember to reserve the indexCommit first so it won't get deleted concurrently
private NamedList createSnapshot(final IndexCommit indexCommit) throws Exception
{
LOG.info("Creating backup snapshot " + (snapshotName == null ? "<not named>" : snapshotName) + " at "
+ baseSnapDirPath);
boolean success = false;
try
{
NamedList<Object> details = new NamedList<>();
details.add("startTime", new Date().toString());//bad; should be Instant.now().toString()
Collection<String> files = indexCommit.getFileNames();
Directory dir = solrCore.getDirectoryFactory()
.get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
try
{
for (String fileName : files)
{
backupRepo.copyFileFrom(dir, fileName, snapshotDirPath);
}
}
finally
{
solrCore.getDirectoryFactory().release(dir);
}
details.add("fileCount", files.size());
details.add("status", "success");
details.add("snapshotCompletedAt", new Date().toString());//bad; should be Instant.now().toString()
details.add("snapshotName", snapshotName);
LOG.info("Done creating backup snapshot: " + (snapshotName == null ? "<not named>" : snapshotName) + " at "
+ baseSnapDirPath);
success = true;
return details;
}
finally
{
if (!success)
{
try
{
backupRepo.deleteDirectory(snapshotDirPath);
}
catch (Exception excDuringDelete)
{
LOG.warn("Failed to delete " + snapshotDirPath + " after snapshot creation failed due to: "
+ excDuringDelete);
}
}
}
}
private void deleteOldBackups(int numberToKeep) throws IOException
{
String[] paths = backupRepo.listAll(baseSnapDirPath);
List<OldBackupDirectory> dirs = new ArrayList<>();
for (String f : paths)
{
if (backupRepo.getPathType(baseSnapDirPath.resolve(f)) == PathType.DIRECTORY)
{
OldBackupDirectory obd = new OldBackupDirectory(baseSnapDirPath, f);
if (obd.getTimestamp().isPresent())
{
dirs.add(obd);
}
}
}
if (numberToKeep > dirs.size() - 1)
{
return;
}
Collections.sort(dirs);
int i = 1;
for (OldBackupDirectory dir : dirs)
{
if (i++ > numberToKeep)
{
backupRepo.deleteDirectory(dir.getPath());
}
}
}
protected void deleteNamedSnapshot(AlfrescoReplicationHandler alfrescoReplicationHandler)
{
LOG.info("Deleting snapshot: " + snapshotName);
NamedList<Object> details = new NamedList<>();
try
{
URI path = baseSnapDirPath.resolve("snapshot." + snapshotName);
backupRepo.deleteDirectory(path);
details.add("status", "success");
details.add("snapshotDeletedAt", new Date().toString());
}
catch (IOException e)
{
details.add("status", "Unable to delete snapshot: " + snapshotName);
LOG.warn("Unable to delete snapshot: " + snapshotName, e);
}
alfrescoReplicationHandler.snapShootDetails = details;
}
private static final String DATE_FMT = "yyyyMMddHHmmssSSS";
}
@@ -113,7 +113,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener
AlfrescoSolrDataModel.getInstance().getDictionaryService(CMISStrictDictionaryService.DEFAULT),
AlfrescoSolrDataModel.getInstance().getNamespaceDAO());
SolrContentStore contentStore = new SolrContentStore(coreContainer.getSolrHome());
SolrContentStore contentStore = admin.getSolrContentStore();
SolrInformationServer informationServer = new SolrInformationServer(admin, core, repositoryClient, contentStore);
coreProperties.putAll(informationServer.getProps());
admin.getInformationServers().put(core.getName(), informationServer);
@@ -162,6 +162,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener
boolean trackersHaveBeenEnabled = Boolean.parseBoolean(coreProperties.getProperty("enable.alfresco.tracking", "true"));
boolean owningCoreIsSlave = isSlaveModeEnabledFor(core);
contentStore.toggleReadOnlyMode(owningCoreIsSlave);
if (trackerRegistry.hasTrackersForCore(core.getName()))
{
@@ -151,6 +151,8 @@ public class CommitTracker extends AbstractTracker
maintenance();
}
infoSrv.flushContentStore();
//Do the commit opening the searcher if needed. This will commit all the work done by indexing trackers.
//This will return immediately and not wait for searchers to warm
//System.out.println("################### Commit:"+openSearcherNeeded);
@@ -0,0 +1,71 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.utils;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Map;
import java.util.stream.Collectors;
/**
* @author Elia Porciani
*/
public class AlfrescoFileUtils {
/**
* Check if two directories contains the same files
*
* @param dir
* @param dir2
* @param extensions Limits the search to the extensions provided
* @param recursive Check recursively in all subdirs
* @return
*/
public static boolean areDirectoryEquals(Path dir, Path dir2, String[] extensions, boolean recursive) {
Map<String, File> filesDir1 = FileUtils.listFiles(new File(dir.toUri()), extensions, recursive)
.stream().collect(Collectors.toMap(f -> f.getName(), f -> f));
Map<String, File> filesDir2 = FileUtils.listFiles(new File(dir2.toUri()), extensions, recursive)
.stream().collect(Collectors.toMap(f -> f.getName(), f -> f));
if (filesDir1.size() != filesDir2.size())
return false;
return filesDir1.entrySet().stream().allMatch(e -> {
File fileDir2 = filesDir2.get(e.getKey());
if (fileDir2 == null) {
return false;
}
try {
byte[] otherBytes = Files.readAllBytes(e.getValue().toPath());
byte[] thisBytes = Files.readAllBytes(fileDir2.toPath());
return (Arrays.equals(otherBytes, thisBytes));
} catch (IOException ex) {
return false;
}
});
}
}
@@ -18,11 +18,18 @@
*/
package org.alfresco.solr.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
public abstract class Utils
{
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
/**
* Returns the same input collection if that is not null, otherwise a new empty collection.
* Provides a safe way for iterating over a returned collection (which could be null).
@@ -53,4 +60,41 @@ public abstract class Utils
return null;
}
}
/**
* Silently closes the given {@link Closeable} resource without raising any exception.
* This utility method is specifically useful when we have to close a resource in a lamba statement: since the
* close() method could throw an {@link IOException} the compiler requires an enclosing try / catch block which
* makes the code less readable.
*
* <br/><br/>
* <p>
* <code>
* try { if (resource != null) resource.close } catch (IOException exception) { ... }
* </code>
* </p>
* <br/>
*
* In these contexts a call to this method reduces the amount of code needed:
*
* <br/><br/>
* <p>
* <code>
* silentlyClose(resource);
* </code>
* </p>
*
* @param resource the {@link Closeable} resource we want to silently close.
*/
public static void silentyClose(Closeable resource)
{
try
{
if (resource != null) resource.close();
}
catch(IOException ignore)
{
LOGGER.warn("Unable to properly close the resource instance {}. See the stacktrace below for further details.", resource, ignore);
}
}
}
@@ -1154,7 +1154,9 @@
https://wiki.apache.org/solr/SolrCloud/
-->
<requestHandler name="/replication" class="solr.ReplicationHandler" >
<requestHandler name="/replication" class="org.alfresco.solr.handler.AlfrescoReplicationHandler">
<!--
To enable simple master/slave replication, uncomment one of the
sections below, depending on whether this solr instance should be
@@ -296,6 +296,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer
{
long begin = System.currentTimeMillis();
//TODO: Support multiple cores per jetty
System.out.println(getStandaloneClients().size());
SolrClient standaloneClient = getStandaloneClients().get(0); //Get the first one
waitForDocCountCore(standaloneClient, query, count, waitMillis, begin);
waitForShardsCount(query, count, waitMillis, begin);
@@ -33,6 +33,7 @@ import org.apache.chemistry.opencmis.commons.impl.json.JSONValue;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.Time;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.solr.SolrTestCaseJ4;
@@ -66,6 +67,7 @@ import org.xml.sax.SAXException;
import javax.servlet.http.HttpServletRequest;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathExpressionException;
import java.io.File;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.file.Paths;
@@ -96,6 +98,12 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
private static Log LOG = LogFactory.getLog(AbstractAlfrescoSolrIT.class);
private static boolean CORE_NOT_YET_CREATED = true;
private static String testExecutionSolrHome;
private static String testSolrCollection;
private static String testSolrConf;
private static String templateConf;
/**
* Harness initialized by initTestHarness.
* <p/>
@@ -118,6 +126,20 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
protected static NodeRef TEST_NODEREF;
protected static Date FTS_TEST_DATE;
protected static void copyTestFiles() throws IOException {
//Add solr home conf folder with alfresco based configuration.
File testExecutionFolder = Paths.get(testExecutionSolrHome).toFile();
for (String s : List.of("/conf", "/alfrescoModels", "/templates", "/collection1")) {
FileUtils.copyDirectory(Paths.get(TEST_FILES_LOCATION + s).toFile(), Paths.get(testExecutionFolder + s).toFile());
}
}
/* Bunch of methods that wrap testHarness object usage.
* TestHarness is a class for internal solr test and should not be use outside Solr.
* Unfortunately this class is used in too many test and a complete refactor would have a huge impact
@@ -217,6 +239,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
@Deprecated
public static void initAlfrescoCore(String schema) throws Exception
{
LOG.info("##################################### init Alfresco core ##############");
LOG.info("####initCore");
System.setProperty("solr.solr.home", TEST_FILES_LOCATION);
@@ -230,18 +253,29 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
System.setProperty("alfresco.test", "true");
System.setProperty("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler");
System.setProperty("solr.tests.mergePolicy", "org.apache.lucene.index.TieredMergePolicy");
if (CORE_NOT_YET_CREATED)
{
testExecutionSolrHome = TEST_EXECUTION_FOLDER + "/" + Time.now() + "/solrhome";
testSolrCollection = testExecutionSolrHome + "/collection1";
testSolrConf = testSolrCollection + "/conf/";
templateConf = testExecutionSolrHome + "/templates/%s/conf/";
copyTestFiles();
createAlfrescoCore(schema);
}
LOG.info("####initCore end");
admin = of(h).map(TestHarness::getCoreContainer)
.map(CoreContainer::getMultiCoreHandler)
.map(AlfrescoCoreAdminHandler.class::cast)
.orElseThrow(RuntimeException::new);
}
/**
* @deprecated as testHarness is used
*/
@@ -262,11 +296,11 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
{
String templateName = System.getProperty("templateName", "rerank");
FileUtils.copyFile(
Paths.get(String.format(TEMPLATE_CONF, templateName) + schema).toFile(),
Paths.get(TEST_SOLR_CONF + schema).toFile());
Paths.get(String.format(templateConf, templateName) + schema).toFile(),
Paths.get(testSolrConf + schema).toFile());
}
SolrResourceLoader resourceLoader = new SolrResourceLoader(Paths.get(TEST_FILES_LOCATION), null, properties);
SolrResourceLoader resourceLoader = new SolrResourceLoader(Paths.get(testExecutionSolrHome), null, properties);
TestCoresLocator locator = new TestCoresLocator(SolrTestCaseJ4.DEFAULT_TEST_CORENAME,
"data",
"solrconfig.xml",
@@ -291,9 +325,9 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
}
@AfterClass()
public static void tearDown()
{
public static void tearDown() throws IOException {
h.close();
FileUtils.forceDelete(Paths.get(TEST_EXECUTION_FOLDER).toFile());
CORE_NOT_YET_CREATED = true;
}
/**
@@ -308,7 +342,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
/**
* Validates an update XML String is successful
*/
public void assertU(String update)
public void assertU(String update)
{
assertU(null, update);
}
@@ -324,7 +358,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
/**
* Validates an update XML String failed
*/
public void assertFailedU(String update)
public void assertFailedU(String update)
{
assertFailedU(null, update);
}
@@ -433,6 +467,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
}
catch (Exception e2)
{
e2.printStackTrace();
throw new RuntimeException("Exception during query", e2);
}
}
@@ -497,7 +532,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
throws Exception
{
Date date = new Date();
long timeout = (long)date.getTime() + waitMillis;
long timeout = date.getTime() + waitMillis;
RefCounted<SolrIndexSearcher> ref = null;
int totalHits = 0;
@@ -737,6 +772,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
}
catch(Exception exception)
{
exception.printStackTrace();
throw new RuntimeException(exception);
}
finally
@@ -784,7 +820,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
{
public SolrServletRequest(SolrCore core, HttpServletRequest req)
{
super(core, new MultiMapSolrParams(Collections.<String, String[]> emptyMap()));
super(core, new MultiMapSolrParams(Collections.emptyMap()));
}
}
@@ -217,9 +217,9 @@ public class AlfrescoSolrUtils
nodeMetaData.setAclId(acl.getId());
nodeMetaData.setTxnId(txn.getId());
nodeMetaData.setOwner(owner);
nodeMetaData.setAspects(new HashSet<QName>());
nodeMetaData.setAspects(new HashSet<>());
nodeMetaData.setAncestors(ancestors);
Map<QName, PropertyValue> props = new HashMap<QName, PropertyValue>();
Map<QName, PropertyValue> props = new HashMap<>();
props.put(ContentModel.PROP_IS_INDEXED, new StringPropertyValue("true"));
props.put(ContentModel.PROP_CONTENT, new ContentPropertyValue(Locale.US, 0l, "UTF-8", "text/plain", null));
nodeMetaData.setProperties(props);
@@ -230,8 +230,8 @@ public class AlfrescoSolrUtils
}
nodeMetaData.setType(QName.createQName(TEST_NAMESPACE, "testSuperType"));
nodeMetaData.setAncestors(ancestors);
nodeMetaData.setPaths(new ArrayList<Pair<String, QName>>());
nodeMetaData.setNamePaths(new ArrayList<List<String>>());
nodeMetaData.setPaths(new ArrayList<>());
nodeMetaData.setNamePaths(new ArrayList<>());
return nodeMetaData;
}
/**
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
@@ -4,6 +4,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import org.alfresco.solr.basics.RandomSupplier;
import org.alfresco.solr.client.SOLRAPIQueueClient;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.util.Time;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.embedded.JettyConfig;
@@ -225,13 +226,14 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
SOLRAPIQueueClient.aclMap.clear();
SOLRAPIQueueClient.nodeMap.clear();
}
/**
* Creates a JettySolrRunner (if one didn't exist already). DOES NOT START IT.
* @return
* @throws Exception
*/
private static JettySolrRunner createJetty(String jettyKey, boolean basicAuth) throws Exception
protected static JettySolrRunner createJetty(String jettyKey, boolean basicAuth) throws Exception
{
if (jettyContainers.containsKey(jettyKey))
{
@@ -241,7 +243,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
{
Path jettySolrHome = testDir.toPath().resolve(jettyKey);
seedSolrHome(jettySolrHome);
JettySolrRunner jetty = createJetty(jettySolrHome.toFile(), null, null, false, getSchemaFile(), basicAuth);
JettySolrRunner jetty = createJetty(jettySolrHome.toFile(), null, null, false, 0, getSchemaFile(), basicAuth);
return jetty;
}
}
@@ -255,7 +257,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
* @param additionalProperties
* @throws Exception
*/
private static void addCoreToJetty(String jettyKey, String sourceConfigName, String coreName, Properties additionalProperties) throws Exception
protected static void addCoreToJetty(String jettyKey, String sourceConfigName, String coreName, Properties additionalProperties) throws Exception
{
Path jettySolrHome = testDir.toPath().resolve(jettyKey);
Path coreSourceConfig = new File(getTestFilesHome() + "/"+sourceConfigName).toPath();
@@ -297,7 +299,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
* @param jsr
* @throws Exception
*/
private static void startJetty(JettySolrRunner jsr) throws Exception {
protected static void startJetty(JettySolrRunner jsr) throws Exception {
if (!jsr.isRunning())
{
jsr.start();
@@ -308,6 +310,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
{
boolean basicAuth = additionalProperties != null ? Boolean.parseBoolean(additionalProperties.getProperty("BasicAuth", "false")) : false;
JettySolrRunner jsr = createJetty(jettyKey, basicAuth);
jettyContainers.put(jettyKey, jsr);
@@ -355,7 +358,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
props.put("shard.range", ranges[i]);
}
String shardKey = jettyKey+"_shard_"+i;
String shardKey = jettyKey+"_shard_" + i + "/solrhome";
JettySolrRunner j = createJetty(shardKey, basicAuth);
//use the first corename specified as the Share template
addCoreToJetty(shardKey, coreNames[0], shardname, props);
@@ -373,7 +376,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
protected static void destroyServers() throws Exception
{
List<String> solrHomes = new ArrayList<String>();
List<String> solrHomes = new ArrayList<>();
for (JettySolrRunner jetty : jettyContainers.values())
{
solrHomes.add(jetty.getSolrHome());
@@ -406,9 +409,9 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
solrCollectionNameToStandaloneClient.clear();
}
public static JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, boolean sslEnabled, String schemaOverride, boolean basicAuth) throws Exception
public static JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, boolean sslEnabled, int port, String schemaOverride, boolean basicAuth) throws Exception
{
return createJetty(solrHome, dataDir, shardList, sslEnabled, schemaOverride, useExplicitNodeNames, basicAuth);
return createJetty(solrHome, dataDir, shardList, sslEnabled, port, schemaOverride, useExplicitNodeNames, basicAuth);
}
/**
@@ -417,12 +420,13 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
* @param solrHome
* @param dataDir
* @param shardList
* @param port
* @param schemaOverride
* @param explicitCoreNodeName
* @return
* @throws Exception
*/
public static JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, boolean sslEnabled,
public static JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, boolean sslEnabled, int port,
String schemaOverride, boolean explicitCoreNodeName, boolean basicAuth) throws Exception
{
Properties props = new Properties();
@@ -445,10 +449,10 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
if(basicAuth) {
System.out.println("###### adding basic auth ######");
config = JettyConfig.builder().setContext("/solr").withFilter(BasicAuthFilter.class, "/sql/*").stopAtShutdown(true).withSSLConfig(sslConfig).build();
config = JettyConfig.builder().setContext("/solr").setPort(port).withFilter(BasicAuthFilter.class, "/sql/*").stopAtShutdown(true).withSSLConfig(sslConfig).build();
} else {
System.out.println("###### no basic auth ######");
config = JettyConfig.builder().setContext("/solr").stopAtShutdown(true).withSSLConfig(sslConfig).build();
config = JettyConfig.builder().setContext("/solr").setPort(port).stopAtShutdown(true).withSSLConfig(sslConfig).build();
}
JettySolrRunner jetty = new JettySolrRunner(solrHome.getAbsolutePath(), props, config);
@@ -24,8 +24,6 @@ package org.alfresco.solr;
*/
public interface SolrTestFiles
{
public final String TEST_FILES_LOCATION = "target/test-classes/test-files";
public final String TEST_SOLR_COLLECTION = TEST_FILES_LOCATION + "/collection1";
public final String TEST_SOLR_CONF = TEST_SOLR_COLLECTION + "/conf/";
public final String TEMPLATE_CONF = TEST_FILES_LOCATION + "/templates/%s/conf/";
String TEST_FILES_LOCATION = "target/test-classes/test-files";
String TEST_EXECUTION_FOLDER = "target/test-execution-folder";
}
@@ -0,0 +1,291 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import java.io.File;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* Solr ContentStore {@link ChangeSet} test case.
*
* @author Andrea Gazzarini
* @since 1.5
*/
@RunWith(MockitoJUnitRunner.class)
public class SolrContentStoreChangeSetTest
{
private ChangeSet changeSet;
private final String contentStoreRootFolder = "/tmp";
private final File rootFolder = new File(contentStoreRootFolder, ChangeSet.CHANGESETS_ROOT_FOLDER_NAME);
@Before
public void setUp()
{
changeSet = new ChangeSet.Builder().withContentStoreRoot(contentStoreRootFolder).build();
}
@After
public void tearDown() throws IOException
{
changeSet.close();
FileUtils.cleanDirectory(rootFolder);
}
@Test(expected = IllegalArgumentException.class)
public void nullContentStoreRootFolder_shouldThrowAnException()
{
new ChangeSet.Builder().withContentStoreRoot(null).build();
}
@Test(expected = IllegalArgumentException.class)
public void nonWriteableContentStoreRootFolder_shouldThrowAnException()
{
new ChangeSet.Builder().withContentStoreRoot("/root").build();
}
@Test
public void newAddOrReplaceRecord_shouldRemovePreviousDeletion()
{
String path ="some/random/dbid.gz";
assertTrue(changeSet.deletes.isEmpty());
assertTrue(changeSet.adds.isEmpty());
changeSet.delete(path);
assertTrue(String.valueOf(changeSet.deletes), changeSet.deletes.contains(path));
assertTrue(String.valueOf(changeSet.adds), changeSet.adds.isEmpty());
changeSet.addOrReplace(path);
assertTrue(String.valueOf(changeSet.deletes), changeSet.deletes.isEmpty());
assertTrue(String.valueOf(changeSet.adds),changeSet.adds.contains(path));
}
@Test
public void deletedRecord_shouldRemovePreviousAdd()
{
String path ="some/random/dbid.gz";
assertTrue(changeSet.deletes.isEmpty());
assertTrue(changeSet.adds.isEmpty());
changeSet.addOrReplace(path);
assertTrue(String.valueOf(changeSet.deletes), changeSet.deletes.isEmpty());
assertTrue(String.valueOf(changeSet.adds), changeSet.adds.contains(path));
changeSet.delete(path);
assertTrue(String.valueOf(changeSet.deletes), changeSet.deletes.contains(path));
assertTrue(String.valueOf(changeSet.adds),changeSet.adds.isEmpty());
}
@Test
public void transientChangeset_doesNothingOnFlush() throws IOException
{
ChangeSet changeset = new ChangeSet.Builder().build();
changeset.addOrReplace("A");
changeset.delete("B");
assertEquals(1, changeset.deletes.size());
assertEquals(1, changeset.adds.size());
changeset.flush();
assertEquals(1, changeset.deletes.size());
assertEquals(1, changeset.adds.size());
}
@Test(expected = UnsupportedOperationException.class)
public void emptyChangeset_isImmutableDoesntAllowAdds()
{
ChangeSet changeset = new ChangeSet.Builder().empty().build();
changeset.addOrReplace("A");
}
@Test(expected = UnsupportedOperationException.class)
public void emptyChangeset_isImmutableDoesntAllowDeletes()
{
ChangeSet changeset = new ChangeSet.Builder().empty().build();
changeset.delete("A");
}
@Test
public void lastCommittedVersionNotPresentAtVeryBeginning()
{
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, changeSet.getLastCommittedVersion());
}
@Test
public void lastCommittedVersionNotAvailable_shouldReturnNO_AVAILABLE_VERSION() throws IOException
{
changeSet.selectEverything = mock(Query.class);
when(changeSet.selectEverything.rewrite(any(IndexReader.class))).thenThrow(new RuntimeException());
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, changeSet.getLastCommittedVersion());
}
@Test
public void flushDoesNothingIfThereAreNoChanges() throws IOException
{
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, changeSet.getLastCommittedVersion());
changeSet.flush();
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, changeSet.getLastCommittedVersion());
}
@Test
public void invalidOrUnknownVersion() throws IOException
{
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, changeSet.getLastCommittedVersion());
assertTrue(changeSet.isUnknownVersion(SolrContentStore.NO_VERSION_AVAILABLE));
assertTrue(changeSet.isUnknownVersion(SolrContentStore.NO_VERSION_AVAILABLE - 1L));
assertTrue(changeSet.isUnknownVersion(System.currentTimeMillis()));
changeSet.addOrReplace("A1");
changeSet.addOrReplace("A2");
changeSet.delete("A3");
changeSet.delete("A1");
changeSet.flush();
long lastCommittedVersionAfterFirstFlush = changeSet.getLastCommittedVersion();
assertNotEquals(SolrContentStore.NO_VERSION_AVAILABLE, lastCommittedVersionAfterFirstFlush);
assertTrue(changeSet.isUnknownVersion(System.currentTimeMillis()));
}
@Test
public void validVersion() throws IOException
{
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, changeSet.getLastCommittedVersion());
assertTrue(changeSet.isUnknownVersion(System.currentTimeMillis()));
changeSet.addOrReplace("A1");
changeSet.addOrReplace("A2");
changeSet.delete("A3");
changeSet.delete("A1");
changeSet.flush();
long lastCommittedVersionAfterFirstFlush = changeSet.getLastCommittedVersion();
changeSet.addOrReplace("B1");
changeSet.addOrReplace("B2");
changeSet.delete("B3");
changeSet.delete("B1");
changeSet.flush();
long lastCommittedVersionAfterSecondFlush = changeSet.getLastCommittedVersion();
assertNotEquals(lastCommittedVersionAfterSecondFlush, lastCommittedVersionAfterFirstFlush);
assertFalse(changeSet.isUnknownVersion(lastCommittedVersionAfterFirstFlush));
assertFalse(changeSet.isUnknownVersion(lastCommittedVersionAfterSecondFlush));
}
@Test
public void inCaseOfFailure_inputVersionIsConsideredUnknown() throws IOException
{
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, changeSet.getLastCommittedVersion());
assertTrue(changeSet.isUnknownVersion(System.currentTimeMillis()));
changeSet.addOrReplace("A1");
changeSet.addOrReplace("A2");
changeSet.delete("A3");
changeSet.delete("A1");
changeSet.flush();
long lastCommittedVersion = changeSet.getLastCommittedVersion();
// Force a NPE exception...
changeSet.searcher = null;
// ...so a valid version is considered unknown even if it is valid
assertTrue(changeSet.isUnknownVersion(lastCommittedVersion));
}
@Test
public void persistentChangesetsAreMergedBeforeReturningToRequestor() throws IOException
{
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, changeSet.getLastCommittedVersion());
changeSet.addOrReplace("A1");
changeSet.addOrReplace("A2");
changeSet.delete("A3");
changeSet.delete("A1");
changeSet.flush();
long lastCommittedVersionAfterFirstFlush = changeSet.getLastCommittedVersion();
assertNotEquals(SolrContentStore.NO_VERSION_AVAILABLE, lastCommittedVersionAfterFirstFlush);
changeSet.addOrReplace("A1");
changeSet.addOrReplace("A3");
changeSet.delete("A4");
changeSet.flush();
long lastCommittedVersionAfterSecondFlush = changeSet.getLastCommittedVersion();
assertNotEquals(lastCommittedVersionAfterFirstFlush, lastCommittedVersionAfterSecondFlush);
ChangeSet changesSinceTheVeryBeginning = changeSet.since(SolrContentStore.NO_VERSION_AVAILABLE);
// ADDS = [A1, A2, A3]
// DELS = [A4]
assertEquals(3, changesSinceTheVeryBeginning.adds.size());
assertEquals(1, changesSinceTheVeryBeginning.deletes.size());
assertTrue(changesSinceTheVeryBeginning.adds.contains("A1"));
assertTrue(changesSinceTheVeryBeginning.adds.contains("A2"));
assertTrue(changesSinceTheVeryBeginning.adds.contains("A3"));
assertTrue(changesSinceTheVeryBeginning.deletes.contains("A4"));
ChangeSet changesAfterSecondFlush = changeSet.since(lastCommittedVersionAfterFirstFlush);
// ADDS = [A1, A3]
// DELS = [A4]
assertEquals(2, changesAfterSecondFlush.adds.size());
assertEquals(1, changesAfterSecondFlush.deletes.size());
assertTrue(changesAfterSecondFlush.adds.contains("A1"));
assertTrue(changesAfterSecondFlush.adds.contains("A3"));
assertTrue(changesAfterSecondFlush.deletes.contains("A4"));
}
}
@@ -18,47 +18,245 @@
*/
package org.alfresco.solr.content;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import org.alfresco.repo.content.ContentContext;
import org.alfresco.service.cmr.repository.ContentReader;
import org.alfresco.service.cmr.repository.ContentWriter;
import org.alfresco.solr.client.NodeMetaData;
import org.apache.commons.io.FileUtils;
import org.apache.solr.common.SolrInputDocument;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
/**
* Tests {@link SolrContentStoreTest}
*
* @author Derek Hulley
* @since 5.0
* @author Andrea Gazzarini
* @since 1.5
*/
@RunWith(MockitoJUnitRunner.class)
public class SolrContentStoreTest
{
private static final String DEFAULT_TENANT = "_DEFAULT_";
private String solrHome = new File("./target/contentstoretest/").getAbsolutePath();
private long dbid = 111;
private String tenant = "me";
private String solrHome;
private SolrContentStore contentStore;
@Before
public void setUp()
{
solrHome = new File("./target/contentstoretest/" + System.currentTimeMillis()).getAbsolutePath();
contentStore = new SolrContentStore(solrHome);
}
@After
public void tearDown() throws IOException
{
contentStore.close();
File rootDir = new File(new SolrContentStore(solrHome).getRootLocation());
FileUtils.deleteDirectory(rootDir);
}
@Test
public void atVeryBeginningAccessModeIsNotSet()
{
assertSame(contentStore.notYetSet, contentStore.currentAccessMode);
}
@Test
public void whenAccessModeIsNotSetMethodCallsThrowsExceptionOrDoNothing()
{
assertSame(contentStore.notYetSet, contentStore.currentAccessMode);
expectIllegalState(contentStore::getLastCommittedVersion);
expectIllegalState(contentStore::setLastCommittedVersion, System.currentTimeMillis());
expectIllegalState(contentStore::getChanges, System.currentTimeMillis());
expectIllegalState(contentStore::removeDocFromContentStore, mock(NodeMetaData.class));
expectIllegalState(contentStore::storeDocOnSolrContentStore, mock(NodeMetaData.class), mock(SolrInputDocument.class));
try
{
contentStore.flushChangeSet();
fail();
}
catch (IOException exception)
{
fail();
}
catch(IllegalStateException expected)
{
// Nothing to be done here
}
try
{
contentStore.storeDocOnSolrContentStore(DEFAULT_TENANT, System.currentTimeMillis(), mock(SolrInputDocument.class));
fail();
}
catch(IllegalStateException expected)
{
// Nothing to be done here
}
}
@Test
public void lastCommittedVersionInReadOnlyModeNotFound()
{
contentStore.toggleReadOnlyMode(true);
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, contentStore.getLastCommittedVersion());
}
@Test
public void lastCommittedVersionInReadOnlyModeNotFoundBecauseException() throws IOException
{
contentStore.toggleReadOnlyMode(true);
Files.write(new File(contentStore.getRootLocation(), ".version").toPath(), "NAN".getBytes());
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, contentStore.getLastCommittedVersion());
}
@Test
public void lastCommittedVersionInReadOnlyModeNotFoundBecauseFileIsEmpty() throws IOException
{
contentStore.toggleReadOnlyMode(true);
File emptyVersionFile = new File(contentStore.getRootLocation(), ".version");
emptyVersionFile.createNewFile();
assertEquals(SolrContentStore.NO_VERSION_AVAILABLE, contentStore.getLastCommittedVersion());
}
@Test
public void getLastCommittedVersionInReadOnlyMode() throws IOException
{
contentStore.toggleReadOnlyMode(true);
long expectedLastCommittedVersion = System.currentTimeMillis();
Files.write(new File(contentStore.getRootLocation(), ".version").toPath(), Long.toString(expectedLastCommittedVersion).getBytes());
assertEquals(expectedLastCommittedVersion, contentStore.getLastCommittedVersion());
}
@Test
public void setLastCommittedVersionInReadOnlyMode()
{
contentStore.toggleReadOnlyMode(true);
long expectedLastCommittedVersion = System.currentTimeMillis();
contentStore.setLastCommittedVersion(expectedLastCommittedVersion);
File versionFile = new File(contentStore.getRootLocation(), ".version");
assertTrue(versionFile.canRead());
assertEquals(expectedLastCommittedVersion, contentStore.getLastCommittedVersion());
}
@Test
public void getChangesInReadOnlyModeReturnsAnEmptyMap()
{
contentStore.toggleReadOnlyMode(true);
assertEquals(Collections.<String, List<Map<String, Object>>>emptyMap(), contentStore.getChanges(System.currentTimeMillis()));
}
@Test
public void transitionFromNotSetToReadOnlyMode()
{
assertSame(contentStore.notYetSet, contentStore.currentAccessMode);
contentStore.toggleReadOnlyMode(true);
assertSame(contentStore.readOnly, contentStore.currentAccessMode);
}
@Test
public void transitionFromNotSetToReadWriteMode()
{
assertSame(contentStore.notYetSet, contentStore.currentAccessMode);
contentStore.toggleReadOnlyMode(false);
assertSame(contentStore.readWrite, contentStore.currentAccessMode);
}
@Test
public void transitionFromReadOnlyToReadWriteMode()
{
assertSame(contentStore.notYetSet, contentStore.currentAccessMode);
contentStore.toggleReadOnlyMode(true);
assertSame(contentStore.readOnly, contentStore.currentAccessMode);
contentStore.toggleReadOnlyMode(false);
assertSame(contentStore.readWrite, contentStore.currentAccessMode);
}
@Test
public void transitionFromReadOnlyToReadOnlyHasNoEffect()
{
assertSame(contentStore.notYetSet, contentStore.currentAccessMode);
contentStore.toggleReadOnlyMode(true);
assertSame(contentStore.readOnly, contentStore.currentAccessMode);
contentStore.toggleReadOnlyMode(true);
assertSame(contentStore.readOnly, contentStore.currentAccessMode);
}
@Test
public void transitionFromReadWriteToReadOnlyModeHasNoEffect()
{
assertSame(contentStore.notYetSet, contentStore.currentAccessMode);
contentStore.toggleReadOnlyMode(false);
assertSame(contentStore.readWrite, contentStore.currentAccessMode);
contentStore.toggleReadOnlyMode(true);
assertSame(contentStore.readWrite, contentStore.currentAccessMode);
}
@Test
public void transitionFromReadWriteToReadWriteHasNoEffect()
{
assertSame(contentStore.notYetSet, contentStore.currentAccessMode);
contentStore.toggleReadOnlyMode(false);
assertSame(contentStore.readWrite, contentStore.currentAccessMode);
contentStore.toggleReadOnlyMode(false);
assertSame(contentStore.readWrite, contentStore.currentAccessMode);
}
@Test(expected = RuntimeException.class)
public void contentStoreCreation_solrHomeNull_shouldThrowException()
{
@@ -117,180 +315,110 @@ public class SolrContentStoreTest
Assert.assertThat(solrContentStore.getRootLocation(), is(solrHome + "/" + SolrContentStore.CONTENT_STORE));
}
/**
* Generates a content context using a string. The URL part will be the same if the
* data provided is the same.
*/
private ContentContext createContentContext(String data)
{
return SolrContentUrlBuilder.start().add("data", data).getContentContext();
}
@Test
public void rootLocation()
{
SolrContentStore store = new SolrContentStore(solrHome);
File rootDir = new File(store.getRootLocation());
Assert.assertTrue(rootDir.exists());
Assert.assertTrue(rootDir.isDirectory());
File rootDir = new File(contentStore.getRootLocation());
assertTrue(rootDir.exists());
assertTrue(rootDir.isDirectory());
}
@Test
public void getWriter()
@Test
public void storeDocOnSolrContentStore()
{
SolrContentStore store = new SolrContentStore(solrHome);
contentStore.toggleReadOnlyMode(false);
ContentContext ctx = createContentContext("abc");
ContentWriter writer = store.getWriter(ctx);
String url = writer.getContentUrl();
SolrInputDocument doc = mock(SolrInputDocument.class);
long dbid = 111;
String tenant = "me";
SolrInputDocument document = contentStore.retrieveDocFromSolrContentStore(tenant, dbid);
Assert.assertNull(document);
Assert.assertNotNull(url);
Assert.assertEquals("URL of the context does not match the writer URL. ", ctx.getContentUrl(), url);
contentStore.storeDocOnSolrContentStore(tenant, dbid, doc);
document = contentStore.retrieveDocFromSolrContentStore(tenant, dbid);
Assert.assertNotNull(document);
}
@Test
public void contentByString()
@Test
public void storeDocOnSolrContentStoreNodeMetaData()
{
SolrContentStore store = new SolrContentStore(solrHome);
contentStore.toggleReadOnlyMode(false);
SolrInputDocument doc = mock(SolrInputDocument.class);
NodeMetaData nodeMetaData = mock(NodeMetaData.class);
SolrInputDocument document = contentStore.retrieveDocFromSolrContentStore(DEFAULT_TENANT, 0);
Assert.assertNull(document);
ContentContext ctx = createContentContext("abc");
ContentWriter writer = store.getWriter(ctx);
contentStore.storeDocOnSolrContentStore(nodeMetaData, doc);
document = contentStore.retrieveDocFromSolrContentStore(DEFAULT_TENANT, 0);
Assert.assertNotNull(document);
}
File file = new File(store.getRootLocation() + "/" + writer.getContentUrl().replace("solr://", ""));
Assert.assertFalse("File was created before anything was written", file.exists());
@Test
public void removeDocFromContentStore()
{
contentStore.toggleReadOnlyMode(false);
SolrInputDocument doc = mock(SolrInputDocument.class);
NodeMetaData nodeMetaData = mock(NodeMetaData.class);
contentStore.storeDocOnSolrContentStore(nodeMetaData, doc);
SolrInputDocument document = contentStore.retrieveDocFromSolrContentStore(DEFAULT_TENANT, 0);
Assert.assertNotNull(document);
String content = "Quick brown fox jumps over the lazy dog.";
writer.putContent(content);
Assert.assertTrue("File was not created.", file.exists());
contentStore.removeDocFromContentStore(nodeMetaData);
document = contentStore.retrieveDocFromSolrContentStore(DEFAULT_TENANT, 0);
Assert.assertNull(document);
}
private void expectIllegalState(Supplier<?> function)
{
try
{
writer.putContent("Should not work");
function.get();
fail();
}
catch (IllegalStateException e)
catch (IllegalStateException expected)
{
// Expected
// Nothing to do, this is expected
}
// Now get the reader
ContentReader reader = store.getReader(ctx.getContentUrl());
Assert.assertNotNull(reader);
Assert.assertTrue(reader.exists());
Assert.assertEquals(content, reader.getContentString());
}
@Test
public void contentByStream() throws Exception
private <T> void expectIllegalState(Consumer<T> function, T arg)
{
SolrContentStore store = new SolrContentStore(solrHome);
ContentContext ctx = createContentContext("abc");
ContentWriter writer = store.getWriter(ctx);
byte[] bytes = new byte[] { 1, 7, 13 };
ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
writer.putContent(bis);
// Now get the reader
ContentReader reader = store.getReader(ctx.getContentUrl());
ByteArrayOutputStream bos = new ByteArrayOutputStream(3);
reader.getContent(bos);
Assert.assertEquals(bytes[0], bos.toByteArray()[0]);
Assert.assertEquals(bytes[1], bos.toByteArray()[1]);
Assert.assertEquals(bytes[2], bos.toByteArray()[2]);
try
{
function.accept(arg);
fail();
}
catch (IllegalStateException expected)
{
// Nothing to do, this is expected
}
}
@Test
public void delete() throws Exception
private <I,O> void expectIllegalState(Function<I,O> function, I arg)
{
SolrContentStore store = new SolrContentStore(solrHome);
ContentContext ctx = createContentContext("abc");
String url = ctx.getContentUrl();
ContentWriter writer = store.getWriter(ctx);
writer.putContent("Content goes here.");
// Check the reader
ContentReader reader = store.getReader(url);
Assert.assertNotNull(reader);
Assert.assertTrue(reader.exists());
// Delete
store.delete(url);
reader = store.getReader(url);
Assert.assertNotNull(reader);
Assert.assertFalse(reader.exists());
// Delete when already gone; should just not fail
store.delete(url);
try
{
function.apply(arg);
fail();
}
catch (IllegalStateException expected)
{
// Nothing to do, this is expected
}
}
/**
* A demonstration of how the store might be used.
*/
@Test
public void exampleUsage()
private <A, B> void expectIllegalState(BiConsumer<A, B> function, A arg1, B arg2)
{
SolrContentStore store = new SolrContentStore(solrHome);
String tenant = "alfresco.com";
long dbId = 12345;
String otherData = "sdfklsfdl";
ContentContext ctxWrite = SolrContentUrlBuilder.start()
.add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(dbId)).add(SolrContentUrlBuilder.KEY_TENANT, tenant)
.add("otherData", otherData).getContentContext();
ContentWriter writer = store.getWriter(ctxWrite);
writer.putContent("a document in plain text");
// The URL can be reliably rebuilt in any order
String urlRead = SolrContentUrlBuilder.start().add("otherData", otherData)
.add(SolrContentUrlBuilder.KEY_TENANT, tenant).add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(dbId))
.get();
ContentReader reader = store.getReader(urlRead);
String documentText = reader.getContentString();
Assert.assertEquals("a document in plain text", documentText);
}
@Test
public void storeDocOnSolrContentStore() throws IOException
{
SolrContentStore solrContentStore = new SolrContentStore(solrHome);
SolrInputDocument doc = Mockito.mock(SolrInputDocument.class);
SolrInputDocument document = solrContentStore.retrieveDocFromSolrContentStore(tenant, dbid);
Assert.assertNull(document);
solrContentStore.storeDocOnSolrContentStore(tenant, dbid, doc);
document = solrContentStore.retrieveDocFromSolrContentStore(tenant, dbid);
Assert.assertNotNull(document);
}
@Test
public void storeDocOnSolrContentStoreNodeMetaData() throws IOException
{
SolrContentStore solrContentStore = new SolrContentStore(solrHome);
SolrInputDocument doc = Mockito.mock(SolrInputDocument.class);
NodeMetaData nodeMetaData = Mockito.mock(NodeMetaData.class);
SolrInputDocument document = solrContentStore.retrieveDocFromSolrContentStore(DEFAULT_TENANT, 0);
Assert.assertNull(document);
solrContentStore.storeDocOnSolrContentStore(nodeMetaData, doc);
document = solrContentStore.retrieveDocFromSolrContentStore(DEFAULT_TENANT, 0);
Assert.assertNotNull(document);
}
@Test
public void removeDocFromContentStore() throws IOException
{
SolrContentStore solrContentStore = new SolrContentStore(solrHome);
SolrInputDocument doc = Mockito.mock(SolrInputDocument.class);
NodeMetaData nodeMetaData = Mockito.mock(NodeMetaData.class);
solrContentStore.storeDocOnSolrContentStore(nodeMetaData, doc);
SolrInputDocument document = solrContentStore.retrieveDocFromSolrContentStore(DEFAULT_TENANT, 0);
Assert.assertNotNull(document);
solrContentStore.removeDocFromContentStore(nodeMetaData);
document = solrContentStore.retrieveDocFromSolrContentStore(DEFAULT_TENANT, 0);
Assert.assertNull(document);
try
{
function.accept(arg1, arg2);
fail();
}
catch (IllegalStateException expected)
{
// Nothing to do, this is expected
}
}
}
@@ -21,7 +21,7 @@ package org.alfresco.solr.content;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.junit.MockitoJUnitRunner;
/**
* Tests {@link SolrContentUrlBuilder}
@@ -85,12 +85,12 @@ public class SolrContentUrlBuilderTest
.add(SolrContentUrlBuilder.KEY_DB_ID, "12345")
.get();
Assert.assertEquals(url1, url1Check);
String url2 = SolrContentUrlBuilder
.start()
.add(SolrContentUrlBuilder.KEY_DB_ID, "123456")
.get();
Assert.assertNotEquals(url1, url2);
Assert.assertEquals("Incorrect URL: " + url1, "solr://default/db/1234/5.gz", url1);
}
@@ -108,7 +108,6 @@ public class SolrContentUrlBuilderTest
.add(SolrContentUrlBuilder.KEY_TENANT, "bob")
.get();
Assert.assertEquals(url1, url1Check);
Assert.assertEquals("Incorrect URL: " + url1, "solr://bob/db/1234/56.gz", url1);
}
@@ -126,7 +125,6 @@ public class SolrContentUrlBuilderTest
.add(SolrContentUrlBuilder.KEY_TENANT, "bob")
.get();
Assert.assertEquals(url1, url1Check);
Assert.assertEquals("Incorrect URL: " + url1, "solr://bob/acl/1234/56.gz", url1);
}
@@ -146,7 +144,6 @@ public class SolrContentUrlBuilderTest
.add(SolrContentUrlBuilder.KEY_TENANT, "bob")
.get();
Assert.assertEquals(url1, url1Check);
Assert.assertEquals("Incorrect URL: " + url1, "solr://bob/db/5432/1.gz", url1);
}
@@ -164,6 +161,7 @@ public class SolrContentUrlBuilderTest
{
// Expected
}
try
{
SolrContentUrlBuilder
@@ -176,6 +174,7 @@ public class SolrContentUrlBuilderTest
{
// Expected
}
try
{
SolrContentUrlBuilder
@@ -188,6 +187,7 @@ public class SolrContentUrlBuilderTest
{
// Expected
}
try
{
SolrContentUrlBuilder
@@ -200,6 +200,7 @@ public class SolrContentUrlBuilderTest
{
// Expected
}
try
{
SolrContentUrlBuilder
@@ -0,0 +1,86 @@
package org.alfresco.solr.content;
import org.alfresco.service.cmr.repository.ContentReader;
import org.alfresco.service.cmr.repository.ContentWriter;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
public class SolrContentWriterTest
{
private String solrHome = new File("./target/contentwriter/").getAbsolutePath();
@After
public void tearDown() throws IOException
{
File rootDir = new File(solrHome);
FileUtils.deleteDirectory(rootDir);
}
private ContentWriter getContentWriter(String name)
{
return new SolrFileContentWriter(new File(solrHome, name), solrHome + "/" + name);
}
private ContentReader getContentReader(String name)
{
return new SolrFileContentReader(new File(solrHome, name), solrHome + "/" + name);
}
@Test
public void contentByString()
{
String filename = "abc";
ContentWriter writer = getContentWriter(filename);
File file = new File(solrHome, filename);
Assert.assertFalse("File was created before anything was written", file.exists());
String content = "Quick brown fox jumps over the lazy dog.";
writer.putContent(content);
Assert.assertTrue("File was not created.", file.exists());
try
{
writer.putContent("Should not work");
}
catch (IllegalStateException e)
{
// Expected
}
// Now get the reader
ContentReader reader = getContentReader(filename);
Assert.assertNotNull(reader);
Assert.assertTrue(reader.exists());
Assert.assertEquals(content, reader.getContentString());
}
@Test
public void contentByStream()
{
String filename = "cbs";
ContentWriter writer = getContentWriter(filename);
byte[] bytes = new byte[] { 1, 7, 13 };
ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
writer.putContent(bis);
// Now get the reader
ContentReader reader = getContentReader(filename);
ByteArrayOutputStream bos = new ByteArrayOutputStream(3);
reader.getContent(bos);
Assert.assertEquals(bytes[0], bos.toByteArray()[0]);
Assert.assertEquals(bytes[1], bos.toByteArray()[1]);
Assert.assertEquals(bytes[2], bos.toByteArray()[2]);
}
}
@@ -0,0 +1,286 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.handler;
import org.alfresco.solr.AbstractAlfrescoDistributedIT;
import org.alfresco.solr.client.Acl;
import org.alfresco.solr.client.AclChangeSet;
import org.alfresco.solr.client.AclReaders;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.client.NodeMetaData;
import org.alfresco.solr.client.SOLRAPIQueueClient;
import org.alfresco.solr.client.Transaction;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.util.Time;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.TermQuery;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
import static java.util.Collections.singletonList;
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
import static org.alfresco.solr.AlfrescoSolrUtils.getNode;
import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData;
import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
import static org.alfresco.solr.utils.AlfrescoFileUtils.areDirectoryEquals;
import static org.carrot2.shaded.guava.common.collect.ImmutableList.of;
/**
* @author Elia Porciani
*
* This test check if the synchronization of contentstore between master and slave is done correctly.
*/
@SolrTestCaseJ4.SuppressSSL
public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT
{
protected static JettySolrRunner master;
protected static JettySolrRunner slave;
protected static SolrClient masterClient;
protected static SolrClient slaveClient;
protected static Path masterSolrHome;
protected static Path slaveSolrHome;
protected static Path masterContentStore;
protected static Path slaveContentStore;
private static Acl acl;
private static final int MILLIS_TIMOUT = 80000;
@BeforeClass
public static void createMasterSlaveEnv() throws Exception
{
Properties properties = new Properties();
clientShards = new ArrayList<>();
solrShards = new ArrayList<>();
solrCollectionNameToStandaloneClient = new HashMap<>();
jettyContainers = new HashMap<>();
String coreName = "master";
boolean basicAuth = Boolean.parseBoolean(properties.getProperty("BasicAuth", "false"));
String masterDir = "master" + Time.now();
String slaveDir = "slave" + Time.now();
String masterKey = masterDir + "/solrHome";
String slaveKey = slaveDir + "/solrHome";
master = createJetty(masterKey, basicAuth);
addCoreToJetty(masterKey, coreName, coreName, null);
startJetty(master);
String slaveCoreName = "slave";
slave = createJetty(slaveKey, basicAuth);
addCoreToJetty(slaveKey, slaveCoreName, slaveCoreName, null);
setMasterUrl(slaveKey, slaveCoreName, master.getBaseUrl().toString() + "/master");
startJetty(slave);
String masterStr = buildUrl(master.getLocalPort()) + "/" + coreName;
String slaveStr = buildUrl(slave.getLocalPort()) + "/" + slaveCoreName;
masterClient = createNewSolrClient(masterStr);
slaveClient = createNewSolrClient(slaveStr);
masterSolrHome = testDir.toPath().resolve(masterKey);
slaveSolrHome = testDir.toPath().resolve(slaveKey);
masterContentStore = testDir.toPath().resolve(masterDir + "/contentstore");
slaveContentStore = testDir.toPath().resolve(slaveDir + "/contentstore");
AclChangeSet aclChangeSet = getAclChangeSet(1);
acl = getAcl(aclChangeSet);
AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
indexAclChangeSet(aclChangeSet,
of(acl),
of(aclReaders));
}
@AfterClass
public static void cleanupMasterSlave() throws Exception
{
master.stop();
slave.stop();
FileUtils.forceDelete(new File(masterSolrHome.getParent().toUri()));
FileUtils.forceDelete(new File(slaveSolrHome.getParent().toUri()));
SOLRAPIQueueClient.nodeMetaDataMap.clear();
SOLRAPIQueueClient.transactionQueue.clear();
SOLRAPIQueueClient.aclChangeSetQueue.clear();
SOLRAPIQueueClient.aclReadersMap.clear();
SOLRAPIQueueClient.aclMap.clear();
SOLRAPIQueueClient.nodeMap.clear();
SOLRAPIQueueClient.nodeContentMap.clear();
}
@Test
public void contentStoreReplicationTest() throws Exception
{
// ADD 250 nodes and check they are replicated
int numNodes = 250;
Transaction bigTxn = getTransaction(0, numNodes);
List<Node> nodes = new ArrayList<>();
List<NodeMetaData> nodeMetaDatas = new ArrayList<>();
for(int i = 0; i<numNodes; i++) {
Node node = getNode(i, bigTxn, acl, Node.SolrApiNodeStatus.UPDATED);
nodes.add(node);
NodeMetaData nodeMetaData = getNodeMetaData(node, bigTxn, acl, "mike", null, false);
node.setNodeRef(nodeMetaData.getNodeRef().toString());
nodeMetaDatas.add(nodeMetaData);
}
indexTransaction(bigTxn, nodes, nodeMetaDatas);
waitForDocCountCore(masterClient,
luceneToSolrQuery(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world"))),
numNodes, MILLIS_TIMOUT, System.currentTimeMillis());
long filesInMasterContentStore = Files.walk(Paths.get(masterContentStore.toUri().resolve("_DEFAULT_")))
.filter(Files::isRegularFile)
.count();
Assert.assertEquals( "master contentStore should have " + numNodes + "files", numNodes, filesInMasterContentStore);
assertTrue("slave content store is not in sync after timeout", waitForContentStoreSync(MILLIS_TIMOUT));
// ADD other 10 nodes
int numUpdates = 10;
int totalNodes = numNodes + numUpdates;
Transaction updateTx = getTransaction(0, numUpdates);
List<Node> updateNodes = new ArrayList<>();
List<NodeMetaData> updateNodeMetaDatas = new ArrayList<>();
for(int i = numNodes; i < totalNodes; i++) {
Node node = getNode(i, updateTx, acl, Node.SolrApiNodeStatus.UPDATED);
updateNodes.add(node);
NodeMetaData nodeMetaData = getNodeMetaData(node, updateTx, acl, "mike", null, false);
node.setNodeRef(nodeMetaData.getNodeRef().toString());
updateNodeMetaDatas.add(nodeMetaData);
}
indexTransaction(updateTx, updateNodes, updateNodeMetaDatas);
waitForDocCountCore(masterClient,
luceneToSolrQuery(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world"))),
numNodes + numUpdates, MILLIS_TIMOUT, System.currentTimeMillis());
filesInMasterContentStore = Files.walk(Paths.get(masterContentStore.toUri().resolve("_DEFAULT_")))
.filter(Files::isRegularFile)
.count();
Assert.assertEquals( "master contentStore should have " + totalNodes + "files", totalNodes, filesInMasterContentStore);
assertTrue("slave content store is not in sync after timeout", waitForContentStoreSync(MILLIS_TIMOUT));
// DELETES 30 nodes
int numDeletes = 30;
Transaction deleteTx = getTransaction(numDeletes, 0);
totalNodes = numNodes + numUpdates - numDeletes;
List<Node> deleteNodes = new ArrayList<>();
List<NodeMetaData> deleteNodeMetaDatas = new ArrayList<>();
for(int i = 0; i<numDeletes; i++) {
Node node = getNode(i, deleteTx, acl, Node.SolrApiNodeStatus.DELETED);
deleteNodes.add(node);
NodeMetaData nodeMetaData = getNodeMetaData(node, deleteTx, acl, "mike", null, false);
node.setNodeRef(nodeMetaData.getNodeRef().toString());
deleteNodeMetaDatas.add(nodeMetaData);
}
indexTransaction(deleteTx, deleteNodes, deleteNodeMetaDatas);
waitForDocCountCore(masterClient,
luceneToSolrQuery(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world"))),
totalNodes, MILLIS_TIMOUT, System.currentTimeMillis());
filesInMasterContentStore = Files.walk(Paths.get(masterContentStore.toUri().resolve("_DEFAULT_")))
.filter(Files::isRegularFile)
.count();
Assert.assertEquals( "master contentStore should have " + totalNodes + "files", totalNodes, filesInMasterContentStore);
assertTrue("slave content store is not in sync after timeout", waitForContentStoreSync(MILLIS_TIMOUT));
}
private static boolean waitForContentStoreSync(long waitMillis) throws InterruptedException
{
long startMillis = System.currentTimeMillis();
long timeout = startMillis + waitMillis;
long increment = 1;
while(new Date().getTime() < timeout)
{
try{
if (areDirectoryEquals(masterContentStore, slaveContentStore, new String[]{"gz"}, true))
{
return true;
}
} catch (Exception e){
// do nothing
}
Thread.sleep(500 * increment);
}
return false;
}
private static void setMasterUrl(String jettyKey, String coreName, String masterUrl) throws IOException
{
Path jettySolrHome = testDir.toPath().resolve(jettyKey);
Path coreHome = jettySolrHome.resolve(coreName);
Path confDir = coreHome.resolve("conf");
Path solrConfigPath = confDir.resolve("solrconfig.xml");
String content = new String(Files.readAllBytes(solrConfigPath));
content = content.replaceAll("\\{masterURL\\}", masterUrl);
Files.write(solrConfigPath, content.getBytes());
}
}
@@ -0,0 +1,14 @@
The folder contains several Solr configuration files that are used within the test suite.
Note that the folder, although it has the structure that Solr expects (<core name>/conf), it is not a complete "core"
folder, because
- some files are missing (e.g. schema.xml)
- some files are used only in specific tests (e.g. solrconfig-rerank.xml, schema-rerank.xml)
During the build process, Maven creates a complete core definition under the test build output folder by merging the
configuration of the "Rerank" template (src/main/resources/templates/rerank) together with the content of this folder.
Note that this folder is copied **after** the template, so duplicates will be overwritten. For example, both folders
have a *solrcore.properties* and a *solrconfig.xml*: the test execution will use are those in this folder (because they
will overwrite the same files in the rerank template).
@@ -0,0 +1,410 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
This is the Solr schema file. This file should be named "schema.xml" and
should be in the conf directory under the solr home
(i.e. ./solr/conf/schema.xml by default)
or located where the classloader for the Solr webapp can find it.
This example schema is the recommended starting point for users.
It should be kept correct and concise, usable out-of-the-box.
For more information, on how to customize this file, please see
http://wiki.apache.org/solr/SchemaXml
-->
<schema name="example" version="1.1">
<!-- attribute "name" is the name of this schema and is only used for display purposes.
Applications should change this to reflect the nature of the search collection.
version="1.1" is Solr's version number for the schema syntax and semantics. It should
not normally be changed by applications.
1.0: multiValued attribute did not exist, all fields are multiValued by nature
1.1: multiValued attribute introduced, false by default -->
<!-- field type definitions. The "name" attribute is
just a label to be used by field definitions. The "class"
attribute and any other attributes determine the real
behavior of the fieldType.
Class names starting with "solr" refer to java classes in the
org.apache.solr.analysis package.
-->
<!-- The StrField type is not analyzed, but indexed/stored verbatim.
- StrField and TextField support an optional compressThreshold which
limits compression (if enabled in the derived fields) to values which
exceed a certain size (in characters).
-->
<fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
<!-- boolean type: "true" or "false" -->
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
<!-- The optional sortMissingLast and sortMissingFirst attributes are
currently supported on types that are sorted internally as strings.
- If sortMissingLast="true", then a sort on this field will cause documents
without the field to come after documents with the field,
regardless of the requested sort order (asc or desc).
- If sortMissingFirst="true", then a sort on this field will cause documents
without the field to come before documents with the field,
regardless of the requested sort order.
- If sortMissingLast="false" and sortMissingFirst="false" (the default),
then default lucene sorting will be used which places docs without the
field first in an ascending sort and last in a descending sort.
-->
<!--
Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
-->
<fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
is a more restricted form of the canonical representation of dateTime
http://www.w3.org/TR/xmlschema-2/#dateTime
The trailing "Z" designates UTC time and is mandatory.
Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
All other components are mandatory.
Expressions can also be used to denote calculations that should be
performed relative to "NOW" to determine the value, ie...
NOW/HOUR
... Round to the start of the current hour
NOW-1DAY
... Exactly 1 day prior to now
NOW/DAY+6MONTHS+3DAYS
... 6 months and 3 days in the future from the start of
the current day
Consult the TrieDateField javadocs for more information.
-->
<fieldType name="date" class="solr.TrieDateField" sortMissingLast="true" omitNorms="true"/>
<!-- The "RandomSortField" is not used to store or search any
data. You can declare fields of this type it in your schema
to generate psuedo-random orderings of your docs for sorting
purposes. The ordering is generated based on the field name
and the version of the index, As long as the index version
remains unchanged, and the same field name is reused,
the ordering of the docs will be consistent.
If you want differend psuedo-random orderings of documents,
for the same version of the index, use a dynamicField and
change the name
-->
<fieldType name="random" class="solr.RandomSortField" indexed="true" />
<!-- solr.TextField allows the specification of custom text analyzers
specified as a tokenizer and a list of token filters. Different
analyzers may be specified for indexing and querying.
The optional positionIncrementGap puts space between multiple fields of
this type on the same document, with the purpose of preventing false phrase
matching across fields.
For more info on customizing your analyzer chain, please see
http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
-->
<!-- One can also specify an existing Analyzer class that has a
default constructor via the class attribute on the analyzer element
<fieldType name="text_greek" class="solr.TextField">
<analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
</fieldType>
-->
<!-- A text field that only splits on whitespace for exact matching of words -->
<fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory"/>
</analyzer>
</fieldType>
<!-- A text field that uses WordDelimiterFilter to enable splitting and matching of
words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
Synonyms and stopwords are customized by external files, and stemming is enabled.
Duplicate tokens at the same position (which may result from Stemmed Synonyms or
WordDelim parts) are removed.
-->
<fieldType name="text" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.MockTokenizerFactory"/>
<!-- in this example, we will only use synonyms at query time
<filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
-->
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
<filter class="solr.PorterStemFilterFactory"/>
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.MockTokenizerFactory"/>
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
<filter class="solr.PorterStemFilterFactory"/>
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
</fieldType>
<fieldType name="text_plain" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.ICUTokenizerFactory"/>
</analyzer>
</fieldType>
<!-- Less flexible matching, but less false matches. Probably not ideal for product names,
but may be good for SKUs. Can insert dashes in the wrong place and still match. -->
<fieldType name="textTight" class="solr.TextField" positionIncrementGap="100" >
<analyzer>
<tokenizer class="solr.MockTokenizerFactory"/>
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
<filter class="solr.EnglishMinimalStemFilterFactory"/>
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
</fieldType>
<!-- This is an example of using the KeywordTokenizer along
With various TokenFilterFactories to produce a sortable field
that does not include some properties of the source text
-->
<fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
<analyzer>
<!-- KeywordTokenizer does no actual tokenizing, so the entire
input string is preserved as a single token
-->
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
<!-- The LowerCase TokenFilter does what you expect, which can be
when you want your sorting to be case insensitive
-->
<filter class="solr.LowerCaseFilterFactory" />
<!-- The TrimFilter removes any leading or trailing whitespace -->
<filter class="solr.TrimFilterFactory" />
<!-- The PatternReplaceFilter gives you the flexibility to use
Java Regular expression to replace any sequence of characters
matching a pattern with an arbitrary replacement string,
which may include back refrences to portions of the orriginal
string matched by the pattern.
See the Java Regular Expression documentation for more
infomation on pattern and replacement string syntax.
http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory"
pattern="([^a-z])" replacement="" replace="all"
/>
</analyzer>
</fieldType>
<!-- since fields of this type are by default not stored or indexed, any data added to
them will be ignored outright
-->
<fieldType name="ignored" stored="false" indexed="false" class="solr.StrField" />
<fieldType name="file" keyField="id" defVal="1" stored="false" indexed="false" class="solr.ExternalFileField" valType="float"/>
<fieldType name="sfile" keyField="sfile_s" defVal="1" stored="false" indexed="false" class="solr.ExternalFileField" valType="float"/>
<fieldType name="tint" class="solr.TrieIntField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tfloat" class="solr.TrieFloatField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tlong" class="solr.TrieLongField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdouble" class="solr.TrieDoubleField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdouble4" class="solr.TrieDoubleField" precisionStep="4" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tints" class="solr.TrieIntField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
<fieldType name="tfloats" class="solr.TrieFloatField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true"/>
<fieldType name="tlongs" class="solr.TrieLongField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true"/>
<fieldType name="tdoubles" class="solr.TrieDoubleField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
<fieldType name="tdates" class="solr.TrieDateField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
<!-- Poly field -->
<fieldType name="xy" class="solr.PointType" dimension="2" subFieldType="double"/>
<fieldType name="xyd" class="solr.PointType" dimension="2" subFieldSuffix="*_d"/>
<fieldType name="geohash" class="solr.GeoHashField"/>
<fieldType name="point" class="solr.PointType" dimension="2" subFieldSuffix="_d"/>
<!-- A specialized field for geospatial search. If indexed, this fieldType must not be multi
valued. -->
<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
<!-- These should pass right through and insure that we can declare external field types -->
<fieldType name="eff_float" keyField="id" defVal="0"
stored="false" indexed="true"
class="solr.ExternalFileField" valType="float"/>
<fieldType name="eff_tfloat" keyField="eff_ti" defVal="0"
stored="false" indexed="true"
class="solr.ExternalFileField" valType="tfloat"/>
<!-- Be sure that the valType can be optional Since valType has done nothing up until now, this is preferred -->
<fieldType name="eff_none" keyField="id" defVal="0"
stored="false" indexed="true"
class="solr.ExternalFileField"/>
<fieldType name="text_no_analyzer" stored="false" indexed="true" class="solr.TextField" />
<fieldType name="text_length" class="solr.TextField">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.StandardFilterFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.LengthFilterFactory" min="2" max="32768"/>
</analyzer>
</fieldType>
<!-- Valid attributes for fields:
name: mandatory - the name for the field
type: mandatory - the name of a previously defined type from the <types> section
indexed: true if this field should be indexed (searchable or sortable)
stored: true if this field should be retrievable
multiValued: true if this field may contain multiple values per document
omitNorms: (expert) set to true to omit the norms associated with
this field (this disables length normalization and index-time
boosting for the field, and saves some memory). Only full-text
fields or fields that need an index-time boost need norms.
termVectors: [false] set to true to store the term vector for a given field.
When using MoreLikeThis, fields used for similarity should be stored for
best performance.
-->
<!-- for testing, a type that does a transform to see if it's correctly done everywhere -->
<field name="id" type="float" indexed="true" stored="true" required="true" />
<field name="FINGERPRINT" type="text_plain" indexed="true" omitNorms="true" stored="false" multiValued="false" required="false" docValues="false"/>
<field name="ACLID" type="long" indexed="true" stored="true" docValues="true" />
<field name="READER" type="string" indexed="true" stored="true" docValues="true" multiValued="true" />
<field name="DENIED" type="string" indexed="true" stored="true" docValues="true" multiValued="true" />
<field name="DOC_TYPE" type="string" indexed="true" stored="true" docValues="true" />
<field name="text" type="text" indexed="true" stored="false" />
<!-- Test a point field for distances -->
<field name="point" type="xy" indexed="true" stored="true" multiValued="false"/>
<field name="pointD" type="xyd" indexed="true" stored="true" multiValued="false"/>
<field name="point_hash" type="geohash" indexed="true" stored="true" multiValued="false"/>
<field name="signatureField" type="string" indexed="true" stored="false"/>
<field name="eff_trie" type="eff_tfloat" />
<field name="text_no_analyzer" type="text_no_analyzer" indexed="true" />
<field name="_version_" type="long" indexed="true" stored="true" multiValued="false" />
<field name="cat" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="cat_docValues" type="string" indexed="true" stored="true" docValues="true" multiValued="true" />
<field name="cat_intDocValues" type="int" indexed="true" stored="true" docValues="true" multiValued="true" />
<field name="cat_floatDocValues" type="float" indexed="true" stored="true" docValues="true" multiValued="true" />
<field name="cat_length" type="text_length" indexed="true" stored="true" multiValued="true"/>
<!-- Dynamic field definitions. If a field name is not found, dynamicFields
will be used if the name matches any of the patterns.
RESTRICTION: the glob-like pattern in the name attribute must have
a "*" only at the start or the end.
EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i)
Longer patterns will be matched first. if equal size patterns
both match, the first appearing in the schema will be used. -->
<dynamicField name="*_s" type="string" indexed="true" stored="true"/>
<dynamicField name="*_s_dv" type="string" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_ss" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_sS" type="string" indexed="false" stored="true"/>
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="*_ii" type="int" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
<dynamicField name="*_f" type="float" indexed="true" stored="true"/>
<dynamicField name="*_d" type="double" indexed="true" stored="true"/>
<dynamicField name="*_ti" type="tint" indexed="true" stored="true"/>
<dynamicField name="*_ti_dv" type="int" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_ti_ni_dv" type="int" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tl" type="tlong" indexed="true" stored="true"/>
<dynamicField name="*_tl_dv" type="tlong" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tl_ni_dv" type="tlong" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tf" type="tfloat" indexed="true" stored="true"/>
<dynamicField name="*_tf_dv" type="tfloat" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tf_ni_dv" type="tfloat" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_td" type="tdouble" indexed="true" stored="true"/>
<dynamicField name="*_td_dv" type="tdouble" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_td_ni_dv" type="tdouble" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tdt" type="tdate" indexed="true" stored="true"/>
<dynamicField name="*_tdt_dv" type="tdate" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tdt_ni_dv" type="tdate" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tis" type="tints" indexed="true" stored="true"/>
<dynamicField name="*_tis_dv" type="tints" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tis_ni_dv" type="tints" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tls" type="tlongs" indexed="true" stored="true"/>
<dynamicField name="*_tls_dv" type="tlongs" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tls_ni_dv" type="tlongs" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tfs" type="tfloats" indexed="true" stored="true"/>
<dynamicField name="*_tfs_dv" type="tfloats" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tfs_ni_dv" type="tfloats" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tds" type="tdoubles" indexed="true" stored="true"/>
<dynamicField name="*_tds_dv" type="tdoubles" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tds_ni_dv" type="tdoubles" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tdts" type="tdates" indexed="true" stored="true"/>
<dynamicField name="*_tdts_dv" type="tdates" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tdts_ni_dv" type="tdates" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="*_ws" type="text_ws" indexed="true" stored="true"/>
<dynamicField name="*_extf" type="file"/>
<dynamicField name="*_extfs" type="sfile"/>
<dynamicField name="*_random" type="random" />
<!-- uncomment the following to ignore any fields that don't already match an existing
field name or dynamic field, rather than reporting them as an error.
alternately, change the type="ignored" to some other type e.g. "text" if you want
unknown fields indexed and/or stored by default -->
<!--dynamicField name="*" type="ignored" /-->
<!-- Field to use to determine and enforce document uniqueness.
Unless this field is marked with required="false", it will be a required field
-->
<uniqueKey>id</uniqueKey>
</schema>
@@ -0,0 +1,47 @@
<?xml version="1.0" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
A solrconfig.xml snippet containing indexConfig settings for randomized testing.
-->
<indexConfig>
<!-- this sys property is not set by SolrTestCaseJ4 because we ideally want to use
the RandomMergePolicy in all tests - but some tests expect very specific
Merge behavior, so those tests can set it as needed.
-->
<mergePolicy class="${solr.tests.mergePolicy:org.apache.solr.util.RandomMergePolicy}" />
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
<maxBufferedDocs>${solr.tests.maxBufferedDocs}</maxBufferedDocs>
<maxIndexingThreads>${solr.tests.maxIndexingThreads}</maxIndexingThreads>
<ramBufferSizeMB>${solr.tests.ramBufferSizeMB}</ramBufferSizeMB>
<mergeScheduler class="${solr.tests.mergeScheduler}" />
<writeLockTimeout>1000</writeLockTimeout>
<commitLockTimeout>10000</commitLockTimeout>
<!-- this sys property is not set by SolrTestCaseJ4 because almost all tests should
use the single process lockType for speed - but tests that explicitly need
to vary the lockType canset it as needed.
-->
<lockType>${solr.tests.lockType:single}</lockType>
</indexConfig>
@@ -0,0 +1,568 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- This is a "kitchen sink" config file that tests can use.
When writting a new test, feel free to add *new* items (plugins,
config options, etc...) as long as they don't break any existing
tests. if you need to test something esoteric please add a new
"solrconfig-your-esoteric-purpose.xml" config file.
Note in particular that this test is used by MinimalSchemaTest so
Anything added to this file needs to work correctly even if there
is now uniqueKey or defaultSearch Field.
-->
<config>
<jmx />
<!-- Used to specify an alternate directory to hold all index data.
It defaults to "index" if not present, and should probably
not be changed if replication is in use. -->
<dataDir>${solr.data.dir:}</dataDir>
<!-- The DirectoryFactory to use for indexes.
solr.StandardDirectoryFactory, the default, is filesystem based.
solr.RAMDirectoryFactory is memory based and not persistent. -->
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}">
<double name="maxWriteMBPerSecDefault">1000000</double>
<double name="maxWriteMBPerSecFlush">2000000</double>
<double name="maxWriteMBPerSecMerge">3000000</double>
<double name="maxWriteMBPerSecRead">4000000</double>
</directoryFactory>
<schemaFactory class="ClassicIndexSchemaFactory"/>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
<updateHandler class="solr.DirectUpdateHandler2">
<!-- autocommit pending docs if certain criteria are met
<autoCommit>
<maxDocs>10000</maxDocs>
<maxTime>3600000</maxTime>
</autoCommit>
-->
<!-- represents a lower bound on the frequency that commits may
occur (in seconds). NOTE: not yet implemented
<commitIntervalLowerBound>0</commitIntervalLowerBound>
-->
<!-- The RunExecutableListener executes an external command.
exe - the name of the executable to run
dir - dir to use as the current working directory. default="."
wait - the calling thread waits until the executable returns. default="true"
args - the arguments to pass to the program. default=nothing
env - environment variables to set. default=nothing
-->
<!-- A postCommit event is fired after every commit
<listener event="postCommit" class="solr.RunExecutableListener">
<str name="exe">/var/opt/resin3/__PORT__/scripts/solr/snapshooter</str>
<str name="dir">/var/opt/resin3/__PORT__</str>
<bool name="wait">true</bool>
<arr name="args"> <str>arg1</str> <str>arg2</str> </arr>
<arr name="env"> <str>MYVAR=val1</str> </arr>
</listener>
-->
<!--
<updateLog enable="${enable.update.log:true}">
<str name="dir">${solr.ulog.dir:}</str>
</updateLog>
<commitWithin>
<softCommit>${solr.commitwithin.softcommit:true}</softCommit>
</commitWithin>
-->
</updateHandler>
<query>
<!-- Maximum number of clauses in a boolean query... can affect
range or wildcard queries that expand to big boolean
queries. An exception is thrown if exceeded.
-->
<maxBooleanClauses>1024</maxBooleanClauses>
<!-- Cache specification for Filters or DocSets - unordered set of *all* documents
that match a particular query.
-->
<filterCache
class="solr.search.FastLRUCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<queryResultCache
class="solr.search.LRUCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<documentCache
class="solr.search.LRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<cache name="perSegFilter"
class="solr.search.LRUCache"
size="10"
initialSize="0"
autowarmCount="10" />
<!-- If true, stored fields that are not requested will be loaded lazily.
-->
<enableLazyFieldLoading>true</enableLazyFieldLoading>
<!--
<cache name="myUserCache"
class="solr.search.LRUCache"
size="4096"
initialSize="1024"
autowarmCount="1024"
regenerator="MyRegenerator"
/>
-->
<!--
<useFilterForSortedQuery>true</useFilterForSortedQuery>
-->
<queryResultWindowSize>10</queryResultWindowSize>
<!-- set maxSize artificially low to exercise both types of sets -->
<HashDocSet maxSize="3" loadFactor="0.75"/>
<!-- boolToFilterOptimizer converts boolean clauses with zero boost
into cached filters if the number of docs selected by the clause exceeds
the threshold (represented as a fraction of the total index)
-->
<boolTofilterOptimizer enabled="false" cacheSize="32" threshold=".05"/>
<!-- a newSearcher event is fired whenever a new searcher is being prepared
and there is a current searcher handling requests (aka registered). -->
<!-- QuerySenderListener takes an array of NamedList and executes a
local query request for each NamedList in sequence. -->
<!--
<listener event="newSearcher" class="solr.QuerySenderListener">
<arr name="queries">
<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>
<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>
</arr>
</listener>
-->
<!-- a firstSearcher event is fired whenever a new searcher is being
prepared but there is no current registered searcher to handle
requests or to gain prewarming data from. -->
<!--
<listener event="firstSearcher" class="solr.QuerySenderListener">
<arr name="queries">
<lst> <str name="q">fast_warm</str> <str name="start">0</str> <str name="rows">10</str> </lst>
</arr>
</listener>
-->
<listener event="firstSearcher" class="org.alfresco.solr.lifecycle.SolrCoreLoadListener" />
</query>
<queryResponseWriter name="xml" default="true"
class="solr.XMLResponseWriter" />
<requestHandler name="/replication" class="org.alfresco.solr.handler.AlfrescoReplicationHandler">
<lst name="master">
<str name="replicateAfter">commit</str>
<str name="confFiles">schema.xml</str>
</lst>
</requestHandler>
<!-- An alternate set representation that uses an integer hash to store filters (sets of docids).
If the set cardinality <= maxSize elements, then HashDocSet will be used instead of the bitset
based HashBitset. -->
<!-- requestHandler plugins... incoming queries will be dispatched to the
correct handler based on the 'qt' param matching the
name of registered handlers.
The "standard" request handler is the default and will be used if qt
is not specified in the request.
-->
<requestHandler name="standard" class="solr.StandardRequestHandler">
<bool name="httpCaching">true</bool>
</requestHandler>
<requestHandler name="/get" class="solr.RealTimeGetHandler">
<lst name="defaults">
<str name="omitHeader">true</str>
</lst>
</requestHandler>
<requestHandler name="dismax" class="solr.SearchHandler" >
<lst name="defaults">
<str name="defType">dismax</str>
<str name="q.alt">*:*</str>
<float name="tie">0.01</float>
<str name="qf">
text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0
</str>
<str name="pf">
text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5
</str>
<str name="bf">
ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3
</str>
<str name="mm">
3&lt;-1 5&lt;-2 6&lt;90%
</str>
<int name="ps">100</int>
</lst>
</requestHandler>
<!-- test query parameter defaults -->
<requestHandler name="defaults" class="solr.StandardRequestHandler">
<lst name="defaults">
<int name="rows">4</int>
<bool name="hl">true</bool>
<str name="hl.fl">text,name,subject,title,whitetok</str>
</lst>
</requestHandler>
<!-- test query parameter defaults -->
<requestHandler name="lazy" class="solr.StandardRequestHandler" startup="lazy">
<lst name="defaults">
<int name="rows">4</int>
<bool name="hl">true</bool>
<str name="hl.fl">text,name,subject,title,whitetok</str>
</lst>
</requestHandler>
<requestHandler name="/update" class="solr.UpdateRequestHandler" />
<searchComponent name="fingerprint" class="org.alfresco.solr.component.FingerPrintComponent"/>
<requestHandler name="/fingerprint" class="org.apache.solr.handler.component.AlfrescoSearchHandler" lazy="true" >
<arr name="components">
<str>fingerprint</str>
</arr>
</requestHandler>
<requestHandler name="/afts" class="org.apache.solr.handler.component.AlfrescoSearchHandler" lazy="true" >
<lst name="defaults">
<str name="defType">afts</str>
<str name="spellcheck">false</str>
<str name="spellcheck.extendedResults">false</str>
<str name="spellcheck.count">5</str>
<str name="spellcheck.alternativeTermCount">2</str>
<str name="spellcheck.maxResultsForSuggest">5</str>
<str name="spellcheck.collate">true</str>
<str name="spellcheck.collateExtendedResults">true</str>
<str name="spellcheck.maxCollationTries">5</str>
<str name="spellcheck.maxCollations">3</str>
<str name="carrot.title">mltext@m___t@{http://www.alfresco.org/model/content/1.0}title</str>
<str name="carrot.url">id</str>
<str name="carrot.snippet">content@s___t@{http://www.alfresco.org/model/content/1.0}content</str>
<bool name="carrot.produceSummary">true</bool>
<bool name="carrot.outputSubClusters">false</bool>
</lst>
<arr name="components">
<str>setLocale</str>
<str>rewriteFacetParameters</str>
<str>query</str>
<str>facet</str>
<str>facet_module</str>
<str>mlt</str>
<str>highlight</str>
<str>stats</str>
<str>debug</str>
<str>clearLocale</str>
<str>rewriteFacetCounts</str>
<str>spellcheck</str>
<str>spellcheckbackcompat</str>
<str>setProcessedDenies</str>
</arr>
<shardHandlerFactory class="org.apache.solr.handler.component.AlfrescoHttpShardHandlerFactory" />
</requestHandler>
<requestHandler name="/native" class="org.apache.solr.handler.component.AlfrescoSearchHandler" lazy="true" >
<!-- default values for query parameters can be specified, these
will be overridden by parameters in the request
-->
<lst name="defaults">
<str name="echoParams">explicit</str>
<int name="rows">10</int>
<str name="df">suggest</str>
</lst>
<arr name="components">
<str>setLocale</str>
<str>query</str>
<str>facet</str>
<str>mlt</str>
<str>highlight</str>
<str>stats</str>
<str>debug</str>
<str>clearLocale</str>
</arr>
<shardHandlerFactory class="org.apache.solr.handler.component.AlfrescoHttpShardHandlerFactory" />
</requestHandler>
<requestHandler name="/cmis" class="org.apache.solr.handler.component.AlfrescoSearchHandler" lazy="true" >
<lst name="defaults">
<str name="defType">cmis</str>
</lst>
<arr name="components">
<str>query</str>
<str>facet</str>
<str>mlt</str>
<str>highlight</str>
<str>stats</str>
<str>debug</str>
</arr>
<shardHandlerFactory class="org.apache.solr.handler.component.AlfrescoHttpShardHandlerFactory" />
</requestHandler>
<searchComponent name="termsComp" class="org.apache.solr.handler.component.TermsComponent"/>
<requestHandler name="/terms" class="org.apache.solr.handler.component.SearchHandler">
<arr name="components">
<str>termsComp</str>
</arr>
</requestHandler>
<requestHandler name="mltrh" class="org.apache.solr.handler.component.SearchHandler">
</requestHandler>
<searchComponent name="tvComponent" class="org.apache.solr.handler.component.TermVectorComponent"/>
<requestHandler name="tvrh" class="org.apache.solr.handler.component.SearchHandler">
<lst name="defaults">
</lst>
<arr name="last-components">
<str>tvComponent</str>
</arr>
</requestHandler>
<requestHandler name="/mlt" class="solr.MoreLikeThisHandler">
</requestHandler>
<searchComponent class="solr.HighlightComponent" name="highlight">
<highlighting class="org.apache.solr.handler.component.AlfrescoSolrHighlighter">
<!-- Configure the standard fragmenter -->
<fragmenter name="gap" class="org.apache.solr.highlight.GapFragmenter" default="true">
<lst name="defaults">
<int name="hl.fragsize">100</int>
</lst>
</fragmenter>
<fragmenter name="regex" class="org.apache.solr.highlight.RegexFragmenter">
<lst name="defaults">
<!-- slightly smaller fragsizes work better because of slop -->
<int name="hl.fragsize">70</int>
<!-- allow 50% slop on fragment sizes -->
<float name="hl.regex.slop">0.5</float>
<!-- a basic sentence pattern -->
<str name="hl.regex.pattern">[-\w ,/\n\&quot;&apos;]{20,200}</str>
</lst>
</fragmenter>
<!-- Configure the standard formatter -->
<formatter name="html" class="org.apache.solr.highlight.HtmlFormatter" default="true">
<lst name="defaults">
<str name="hl.simple.pre"><![CDATA[<em>]]></str>
<str name="hl.simple.post"><![CDATA[</em>]]></str>
</lst>
</formatter>
<!-- Configure the standard encoder -->
<encoder name="html" class="solr.highlight.HtmlEncoder"/>
<!-- Configure the standard fragListBuilder -->
<fragListBuilder name="simple" class="solr.highlight.SimpleFragListBuilder"/>
<!-- Configure the single fragListBuilder -->
<fragListBuilder name="single" class="solr.highlight.SingleFragListBuilder"/>
<!-- Configure the weighted fragListBuilder -->
<fragListBuilder name="weighted" default="true" class="solr.highlight.WeightedFragListBuilder"/>
<!-- default tag FragmentsBuilder -->
<fragmentsBuilder name="default" default="true" class="solr.highlight.ScoreOrderFragmentsBuilder">
<!--
<lst name="defaults">
<str name="hl.multiValuedSeparatorChar">/</str>
</lst>
-->
</fragmentsBuilder>
<!-- multi-colored tag FragmentsBuilder -->
<fragmentsBuilder name="colored" class="solr.highlight.ScoreOrderFragmentsBuilder">
<lst name="defaults">
<str name="hl.tag.pre"><![CDATA[
<b style="background:yellow">,<b style="background:lawgreen">,
<b style="background:aquamarine">,<b style="background:magenta">,
<b style="background:palegreen">,<b style="background:coral">,
<b style="background:wheat">,<b style="background:khaki">,
<b style="background:lime">,<b style="background:deepskyblue">]]></str>
<str name="hl.tag.post"><![CDATA[</b>]]></str>
</lst>
</fragmentsBuilder>
<boundaryScanner name="default" default="true" class="solr.highlight.SimpleBoundaryScanner">
<lst name="defaults">
<str name="hl.bs.maxScan">10</str>
<str name="hl.bs.chars">.,!? &#9;&#10;&#13;</str>
</lst>
</boundaryScanner>
<boundaryScanner name="breakIterator" class="solr.highlight.BreakIteratorBoundaryScanner">
<lst name="defaults">
<!-- type should be one of CHARACTER, WORD(default), LINE and SENTENCE -->
<str name="hl.bs.type">WORD</str>
<!-- language and country are used when constructing Locale object. -->
<!-- And the Locale object will be used when getting instance of BreakIterator -->
<str name="hl.bs.language">en</str>
<str name="hl.bs.country">US</str>
</lst>
</boundaryScanner>
</highlighting>
</searchComponent>
<!-- enable streaming for testing... -->
<requestDispatcher handleSelect="true" >
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048" />
<httpCaching lastModifiedFrom="openTime" etagSeed="Solr" never304="false">
<cacheControl>max-age=30, public</cacheControl>
</httpCaching>
</requestDispatcher>
<!-- Echo the request contents back to the client -->
<requestHandler name="/debug/dump" class="solr.DumpRequestHandler" >
<lst name="defaults">
<str name="echoParams">explicit</str>
<str name="echoHandler">true</str>
</lst>
</requestHandler>
<admin>
<defaultQuery>solr</defaultQuery>
<gettableFiles>solrconfig.xml schema.xml admin-extra.html</gettableFiles>
</admin>
<!-- test getting system property -->
<!--
<propTest attr1="${solr.test.sys.prop1}-$${literal}"
attr2="${non.existent.sys.prop:default-from-config}">prefix-${solr.test.sys.prop2}-suffix</propTest>
-->
<queryParser name="alfrescoReRank" class="org.alfresco.solr.query.AlfrescoReRankQParserPlugin"/>
<queryParser name="afts" class="org.alfresco.solr.query.AlfrescoFTSQParserPlugin"/>
<!--
<queryParser name="afts" class="org.alfresco.solr.query.AlfrescoFTSQParserPlugin">
<str name="rerankPhase">SINGLE_PASS</str>
</queryParser>
-->
<queryParser name="cmis" class="org.alfresco.solr.query.CmisQParserPlugin"/>
<xi:include href="solrconfig_insight.xml" xmlns:xi="http://www.w3.org/2001/XInclude">
<xi:fallback/>
</xi:include>
<queryParser name="mimetype" class="org.alfresco.solr.query.MimetypeGroupingQParserPlugin" >
<str name="mapping">conf/mime_types.csv</str>
</queryParser>
<queryParser name="contentSize" class="org.alfresco.solr.query.ContentSizeGroupingQParserPlugin" >
<int name="scale">1</int>
<int name="buckets">10</int>
</queryParser>
<searchComponent name="setLocale" class="org.alfresco.solr.component.SetLocaleComponent" />
<searchComponent name="clearLocale" class="org.alfresco.solr.component.ClearLocaleComponent" />
<!-- Spell Check
The spell check component can return a list of alternative spelling
suggestions.
http://wiki.apache.org/solr/SpellCheckComponent
-->
<searchComponent name="spellcheck" class="org.alfresco.solr.component.spellcheck.AlfrescoSpellCheckComponent">
<str name="queryAnalyzerFieldType">text_shingle</str>
<!-- Multiple "Spell Checkers" can be declared and used by this
component
-->
<!-- a spellchecker built from a field of the main index -->
<lst name="spellchecker">
<str name="name">default</str>
<str name="field">suggest</str>
<str name="classname">solr.DirectSolrSpellChecker</str>
<!-- the spellcheck distance measure used, the default is the internal levenshtein -->
<str name="distanceMeasure">internal</str>
<!-- minimum accuracy needed to be considered a valid spellcheck suggestion -->
<float name="accuracy">0.5</float>
<!-- the maximum #edits we consider when enumerating terms: can be 1 or 2 -->
<int name="maxEdits">2</int>
<!-- the minimum shared prefix when enumerating terms -->
<int name="minPrefix">1</int>
<!-- maximum number of inspections per result. -->
<int name="maxInspections">5</int>
<!-- minimum length of a query term to be considered for correction -->
<int name="minQueryLength">4</int>
<!-- maximum threshold of documents a query term can appear to be considered for correction -->
<float name="maxQueryFrequency">0.01</float>
<!-- uncomment this to require suggestions to occur in 1% of the documents
<float name="thresholdTokenFrequency">.01</float>
-->
</lst>
<!-- word break -->
<lst name="spellchecker">
<str name="name">wordbreak</str>
<str name="field">suggest</str>
<str name="classname">solr.WordBreakSolrSpellChecker</str>
<str name="combineWords">true</str>
<str name="breakWords">true</str>
<int name="maxChanges">10</int>
<int name="minBreakLength">5</int>
</lst>
</searchComponent>
<searchComponent name="spellcheckbackcompat" class="org.alfresco.solr.component.spellcheck.AlfrescoSpellCheckBackCompatComponent"/>
<searchComponent name="rewriteFacetParameters" class="org.alfresco.solr.component.RewriteFacetParametersComponent" />
<searchComponent name="rewriteFacetCounts" class="org.alfresco.solr.component.RewriteFacetCountsComponent" />
<searchComponent name="setProcessedDenies" class="org.alfresco.solr.component.SetProcessedDeniesComponent" />
<transformer name="cached" class="org.alfresco.solr.transformer.CachedDocTransformerFactory" >
</transformer>
</config>
@@ -0,0 +1,154 @@
#
# solrcore.properties - used in solrconfig.xml
#
enable.alfresco.tracking=true
#
#These are replaced by the admin handler
#
#data.dir.root=DATA_DIR
#data.dir.store=workspace/SpacesStore
#alfresco.stores=workspace://SpacesStore
#
# Properties loaded during alfresco tracking
#
alfresco.host=localhost
alfresco.port=8080
alfresco.port.ssl=8443
alfresco.baseUrl=/alfresco
alfresco.cron=0/2 * * * * ? *
#alfresco.index.transformContent=false
#alfresco.ignore.datatype.1=d:content
alfresco.lag=1000
alfresco.hole.retention=3600000
# alfresco.hole.check.after is not used yet
# It will reduce the hole checking load
alfresco.hole.check.after=300000
alfresco.batch.count=1000
# encryption
# none, https
alfresco.secureComms=none
# ssl
alfresco.encryption.ssl.keystore.type=JCEKS
alfresco.encryption.ssl.keystore.provider=
alfresco.encryption.ssl.keystore.location=ssl.repo.client.keystore
alfresco.encryption.ssl.keystore.passwordFileLocation=ssl-keystore-passwords.properties
alfresco.encryption.ssl.truststore.type=JCEKS
alfresco.encryption.ssl.truststore.provider=
alfresco.encryption.ssl.truststore.location=ssl.repo.client.truststore
alfresco.encryption.ssl.truststore.passwordFileLocation=ssl-truststore-passwords.properties
# Tracking
alfresco.corePoolSize=1
alfresco.maximumPoolSize=-1
alfresco.keepAliveTime=120
alfresco.threadPriority=5
alfresco.threadDaemon=true
alfresco.workQueueSize=-1
# HTTP Client
alfresco.maxTotalConnections=200
alfresco.maxHostConnections=200
alfresco.socketTimeout=360000
# SOLR caching
solr.filterCache.size=256
solr.filterCache.initialSize=128
solr.queryResultCache.size=1024
solr.queryResultCache.initialSize=1024
solr.documentCache.size=1024
solr.documentCache.initialSize=1024
solr.queryResultMaxDocsCached=2048
solr.authorityCache.size=128
solr.authorityCache.initialSize=64
solr.pathCache.size=256
solr.pathCache.initialSize=128
solr.ownerCache.size=128
solr.ownerCache.initialSize=64
solr.readerCache.size=128
solr.readerCache.initialSize=64
solr.deniedCache.size=128
solr.deniedCache.initialSize=64
# SOLR
solr.maxBooleanClauses=10000
# Batch fetch
alfresco.transactionDocsBatchSize=100
alfresco.nodeBatchSize=10
alfresco.changeSetAclsBatchSize=100
alfresco.aclBatchSize=10
alfresco.contentReadBatchSize=4000
alfresco.contentUpdateBatchSize=1000
# Warming
solr.filterCache.autowarmCount=32
solr.authorityCache.autowarmCount=4
solr.pathCache.autowarmCount=32
solr.deniedCache.autowarmCount=0
solr.readerCache.autowarmCount=0
solr.ownerCache.autowarmCount=0
solr.queryResultCache.autowarmCount=4
solr.documentCache.autowarmCount=512
solr.queryResultWindowSize=512
#
# TODO
#
# cross language support
# locale expansion
# logging check report ....
#
#
alfresco.commitInterval=1000
alfresco.newSearcherInterval=2000
alfresco.doPermissionChecks=true
#
# Metadata pulling control
#
alfresco.metadata.skipDescendantDocsForSpecificTypes=false
alfresco.metadata.ignore.datatype.0=cm:person
alfresco.metadata.ignore.datatype.1=app:configurations
alfresco.metadata.skipDescendantDocsForSpecificAspects=false
#alfresco.metadata.ignore.aspect.0=
#
# Suggestions
#
solr.suggester.enabled=false
# -1 to disable suggester build throttling
solr.suggester.minSecsBetweenBuilds=3600
#
# Limit the maximum text size of transformed content sent to the index - in bytes
#
alfresco.contentStreamLimit=10000000
#Sharding default values
shard.instance=1
shard.count=0
shard.method=DB_ID
@@ -0,0 +1,50 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
a
an
and
are
as
at
be
but
by
for
if
in
into
is
it
no
not
of
on
or
s
such
t
that
the
their
then
there
these
they
this
to
was
will
with
@@ -0,0 +1,35 @@
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#-----------------------------------------------------------------------
#some test synonym mappings unlikely to appear in real input text
aaa => aaaa
bbb => bbbb1 bbbb2
ccc => cccc1,cccc2
a\=>a => b\=>b
a\,a => b\,b
fooaaa,baraaa,bazaaa
# Some synonym groups specific to this example
GB,gib,gigabyte,gigabytes
MB,mib,megabyte,megabytes
Television, Televisions, TV, TVs
#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming
#after us won't split it into two words.
# Synonym mappings can be used for spelling correction too
pixima => pixma
# Test synonyms
quick,fast,rapid,speedy
brown fox jumped,leaping reynard,springer
lazy,bone idle
@@ -0,0 +1,14 @@
The folder contains several Solr configuration files that are used within the test suite.
Note that the folder, although it has the structure that Solr expects (<core name>/conf), it is not a complete "core"
folder, because
- some files are missing (e.g. schema.xml)
- some files are used only in specific tests (e.g. solrconfig-rerank.xml, schema-rerank.xml)
During the build process, Maven creates a complete core definition under the test build output folder by merging the
configuration of the "Rerank" template (src/main/resources/templates/rerank) together with the content of this folder.
Note that this folder is copied **after** the template, so duplicates will be overwritten. For example, both folders
have a *solrcore.properties* and a *solrconfig.xml*: the test execution will use are those in this folder (because they
will overwrite the same files in the rerank template).
@@ -0,0 +1,410 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
This is the Solr schema file. This file should be named "schema.xml" and
should be in the conf directory under the solr home
(i.e. ./solr/conf/schema.xml by default)
or located where the classloader for the Solr webapp can find it.
This example schema is the recommended starting point for users.
It should be kept correct and concise, usable out-of-the-box.
For more information, on how to customize this file, please see
http://wiki.apache.org/solr/SchemaXml
-->
<schema name="example" version="1.1">
<!-- attribute "name" is the name of this schema and is only used for display purposes.
Applications should change this to reflect the nature of the search collection.
version="1.1" is Solr's version number for the schema syntax and semantics. It should
not normally be changed by applications.
1.0: multiValued attribute did not exist, all fields are multiValued by nature
1.1: multiValued attribute introduced, false by default -->
<!-- field type definitions. The "name" attribute is
just a label to be used by field definitions. The "class"
attribute and any other attributes determine the real
behavior of the fieldType.
Class names starting with "solr" refer to java classes in the
org.apache.solr.analysis package.
-->
<!-- The StrField type is not analyzed, but indexed/stored verbatim.
- StrField and TextField support an optional compressThreshold which
limits compression (if enabled in the derived fields) to values which
exceed a certain size (in characters).
-->
<fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
<!-- boolean type: "true" or "false" -->
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
<!-- The optional sortMissingLast and sortMissingFirst attributes are
currently supported on types that are sorted internally as strings.
- If sortMissingLast="true", then a sort on this field will cause documents
without the field to come after documents with the field,
regardless of the requested sort order (asc or desc).
- If sortMissingFirst="true", then a sort on this field will cause documents
without the field to come before documents with the field,
regardless of the requested sort order.
- If sortMissingLast="false" and sortMissingFirst="false" (the default),
then default lucene sorting will be used which places docs without the
field first in an ascending sort and last in a descending sort.
-->
<!--
Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
-->
<fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
is a more restricted form of the canonical representation of dateTime
http://www.w3.org/TR/xmlschema-2/#dateTime
The trailing "Z" designates UTC time and is mandatory.
Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
All other components are mandatory.
Expressions can also be used to denote calculations that should be
performed relative to "NOW" to determine the value, ie...
NOW/HOUR
... Round to the start of the current hour
NOW-1DAY
... Exactly 1 day prior to now
NOW/DAY+6MONTHS+3DAYS
... 6 months and 3 days in the future from the start of
the current day
Consult the TrieDateField javadocs for more information.
-->
<fieldType name="date" class="solr.TrieDateField" sortMissingLast="true" omitNorms="true"/>
<!-- The "RandomSortField" is not used to store or search any
data. You can declare fields of this type it in your schema
to generate psuedo-random orderings of your docs for sorting
purposes. The ordering is generated based on the field name
and the version of the index, As long as the index version
remains unchanged, and the same field name is reused,
the ordering of the docs will be consistent.
If you want differend psuedo-random orderings of documents,
for the same version of the index, use a dynamicField and
change the name
-->
<fieldType name="random" class="solr.RandomSortField" indexed="true" />
<!-- solr.TextField allows the specification of custom text analyzers
specified as a tokenizer and a list of token filters. Different
analyzers may be specified for indexing and querying.
The optional positionIncrementGap puts space between multiple fields of
this type on the same document, with the purpose of preventing false phrase
matching across fields.
For more info on customizing your analyzer chain, please see
http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
-->
<!-- One can also specify an existing Analyzer class that has a
default constructor via the class attribute on the analyzer element
<fieldType name="text_greek" class="solr.TextField">
<analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
</fieldType>
-->
<!-- A text field that only splits on whitespace for exact matching of words -->
<fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory"/>
</analyzer>
</fieldType>
<!-- A text field that uses WordDelimiterFilter to enable splitting and matching of
words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
Synonyms and stopwords are customized by external files, and stemming is enabled.
Duplicate tokens at the same position (which may result from Stemmed Synonyms or
WordDelim parts) are removed.
-->
<fieldType name="text" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.MockTokenizerFactory"/>
<!-- in this example, we will only use synonyms at query time
<filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
-->
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
<filter class="solr.PorterStemFilterFactory"/>
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.MockTokenizerFactory"/>
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
<filter class="solr.PorterStemFilterFactory"/>
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
</fieldType>
<fieldType name="text_plain" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.ICUTokenizerFactory"/>
</analyzer>
</fieldType>
<!-- Less flexible matching, but less false matches. Probably not ideal for product names,
but may be good for SKUs. Can insert dashes in the wrong place and still match. -->
<fieldType name="textTight" class="solr.TextField" positionIncrementGap="100" >
<analyzer>
<tokenizer class="solr.MockTokenizerFactory"/>
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
<filter class="solr.EnglishMinimalStemFilterFactory"/>
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
</fieldType>
<!-- This is an example of using the KeywordTokenizer along
With various TokenFilterFactories to produce a sortable field
that does not include some properties of the source text
-->
<fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
<analyzer>
<!-- KeywordTokenizer does no actual tokenizing, so the entire
input string is preserved as a single token
-->
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
<!-- The LowerCase TokenFilter does what you expect, which can be
when you want your sorting to be case insensitive
-->
<filter class="solr.LowerCaseFilterFactory" />
<!-- The TrimFilter removes any leading or trailing whitespace -->
<filter class="solr.TrimFilterFactory" />
<!-- The PatternReplaceFilter gives you the flexibility to use
Java Regular expression to replace any sequence of characters
matching a pattern with an arbitrary replacement string,
which may include back refrences to portions of the orriginal
string matched by the pattern.
See the Java Regular Expression documentation for more
infomation on pattern and replacement string syntax.
http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory"
pattern="([^a-z])" replacement="" replace="all"
/>
</analyzer>
</fieldType>
<!-- since fields of this type are by default not stored or indexed, any data added to
them will be ignored outright
-->
<fieldType name="ignored" stored="false" indexed="false" class="solr.StrField" />
<fieldType name="file" keyField="id" defVal="1" stored="false" indexed="false" class="solr.ExternalFileField" valType="float"/>
<fieldType name="sfile" keyField="sfile_s" defVal="1" stored="false" indexed="false" class="solr.ExternalFileField" valType="float"/>
<fieldType name="tint" class="solr.TrieIntField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tfloat" class="solr.TrieFloatField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tlong" class="solr.TrieLongField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdouble" class="solr.TrieDoubleField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdouble4" class="solr.TrieDoubleField" precisionStep="4" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tints" class="solr.TrieIntField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
<fieldType name="tfloats" class="solr.TrieFloatField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true"/>
<fieldType name="tlongs" class="solr.TrieLongField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true"/>
<fieldType name="tdoubles" class="solr.TrieDoubleField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
<fieldType name="tdates" class="solr.TrieDateField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
<!-- Poly field -->
<fieldType name="xy" class="solr.PointType" dimension="2" subFieldType="double"/>
<fieldType name="xyd" class="solr.PointType" dimension="2" subFieldSuffix="*_d"/>
<fieldType name="geohash" class="solr.GeoHashField"/>
<fieldType name="point" class="solr.PointType" dimension="2" subFieldSuffix="_d"/>
<!-- A specialized field for geospatial search. If indexed, this fieldType must not be multi
valued. -->
<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
<!-- These should pass right through and insure that we can declare external field types -->
<fieldType name="eff_float" keyField="id" defVal="0"
stored="false" indexed="true"
class="solr.ExternalFileField" valType="float"/>
<fieldType name="eff_tfloat" keyField="eff_ti" defVal="0"
stored="false" indexed="true"
class="solr.ExternalFileField" valType="tfloat"/>
<!-- Be sure that the valType can be optional Since valType has done nothing up until now, this is preferred -->
<fieldType name="eff_none" keyField="id" defVal="0"
stored="false" indexed="true"
class="solr.ExternalFileField"/>
<fieldType name="text_no_analyzer" stored="false" indexed="true" class="solr.TextField" />
<fieldType name="text_length" class="solr.TextField">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.StandardFilterFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.LengthFilterFactory" min="2" max="32768"/>
</analyzer>
</fieldType>
<!-- Valid attributes for fields:
name: mandatory - the name for the field
type: mandatory - the name of a previously defined type from the <types> section
indexed: true if this field should be indexed (searchable or sortable)
stored: true if this field should be retrievable
multiValued: true if this field may contain multiple values per document
omitNorms: (expert) set to true to omit the norms associated with
this field (this disables length normalization and index-time
boosting for the field, and saves some memory). Only full-text
fields or fields that need an index-time boost need norms.
termVectors: [false] set to true to store the term vector for a given field.
When using MoreLikeThis, fields used for similarity should be stored for
best performance.
-->
<!-- for testing, a type that does a transform to see if it's correctly done everywhere -->
<field name="id" type="float" indexed="true" stored="true" required="true" />
<field name="FINGERPRINT" type="text_plain" indexed="true" omitNorms="true" stored="false" multiValued="false" required="false" docValues="false"/>
<field name="ACLID" type="long" indexed="true" stored="true" docValues="true" />
<field name="READER" type="string" indexed="true" stored="true" docValues="true" multiValued="true" />
<field name="DENIED" type="string" indexed="true" stored="true" docValues="true" multiValued="true" />
<field name="DOC_TYPE" type="string" indexed="true" stored="true" docValues="true" />
<field name="text" type="text" indexed="true" stored="false" />
<!-- Test a point field for distances -->
<field name="point" type="xy" indexed="true" stored="true" multiValued="false"/>
<field name="pointD" type="xyd" indexed="true" stored="true" multiValued="false"/>
<field name="point_hash" type="geohash" indexed="true" stored="true" multiValued="false"/>
<field name="signatureField" type="string" indexed="true" stored="false"/>
<field name="eff_trie" type="eff_tfloat" />
<field name="text_no_analyzer" type="text_no_analyzer" indexed="true" />
<field name="_version_" type="long" indexed="true" stored="true" multiValued="false" />
<field name="cat" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="cat_docValues" type="string" indexed="true" stored="true" docValues="true" multiValued="true" />
<field name="cat_intDocValues" type="int" indexed="true" stored="true" docValues="true" multiValued="true" />
<field name="cat_floatDocValues" type="float" indexed="true" stored="true" docValues="true" multiValued="true" />
<field name="cat_length" type="text_length" indexed="true" stored="true" multiValued="true"/>
<!-- Dynamic field definitions. If a field name is not found, dynamicFields
will be used if the name matches any of the patterns.
RESTRICTION: the glob-like pattern in the name attribute must have
a "*" only at the start or the end.
EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i)
Longer patterns will be matched first. if equal size patterns
both match, the first appearing in the schema will be used. -->
<dynamicField name="*_s" type="string" indexed="true" stored="true"/>
<dynamicField name="*_s_dv" type="string" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_ss" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_sS" type="string" indexed="false" stored="true"/>
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="*_ii" type="int" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
<dynamicField name="*_f" type="float" indexed="true" stored="true"/>
<dynamicField name="*_d" type="double" indexed="true" stored="true"/>
<dynamicField name="*_ti" type="tint" indexed="true" stored="true"/>
<dynamicField name="*_ti_dv" type="int" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_ti_ni_dv" type="int" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tl" type="tlong" indexed="true" stored="true"/>
<dynamicField name="*_tl_dv" type="tlong" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tl_ni_dv" type="tlong" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tf" type="tfloat" indexed="true" stored="true"/>
<dynamicField name="*_tf_dv" type="tfloat" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tf_ni_dv" type="tfloat" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_td" type="tdouble" indexed="true" stored="true"/>
<dynamicField name="*_td_dv" type="tdouble" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_td_ni_dv" type="tdouble" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tdt" type="tdate" indexed="true" stored="true"/>
<dynamicField name="*_tdt_dv" type="tdate" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tdt_ni_dv" type="tdate" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tis" type="tints" indexed="true" stored="true"/>
<dynamicField name="*_tis_dv" type="tints" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tis_ni_dv" type="tints" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tls" type="tlongs" indexed="true" stored="true"/>
<dynamicField name="*_tls_dv" type="tlongs" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tls_ni_dv" type="tlongs" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tfs" type="tfloats" indexed="true" stored="true"/>
<dynamicField name="*_tfs_dv" type="tfloats" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tfs_ni_dv" type="tfloats" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tds" type="tdoubles" indexed="true" stored="true"/>
<dynamicField name="*_tds_dv" type="tdoubles" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tds_ni_dv" type="tdoubles" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_tdts" type="tdates" indexed="true" stored="true"/>
<dynamicField name="*_tdts_dv" type="tdates" indexed="true" stored="true" docValues="true"/>
<dynamicField name="*_tdts_ni_dv" type="tdates" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="*_ws" type="text_ws" indexed="true" stored="true"/>
<dynamicField name="*_extf" type="file"/>
<dynamicField name="*_extfs" type="sfile"/>
<dynamicField name="*_random" type="random" />
<!-- uncomment the following to ignore any fields that don't already match an existing
field name or dynamic field, rather than reporting them as an error.
alternately, change the type="ignored" to some other type e.g. "text" if you want
unknown fields indexed and/or stored by default -->
<!--dynamicField name="*" type="ignored" /-->
<!-- Field to use to determine and enforce document uniqueness.
Unless this field is marked with required="false", it will be a required field
-->
<uniqueKey>id</uniqueKey>
</schema>
@@ -0,0 +1,47 @@
<?xml version="1.0" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
A solrconfig.xml snippet containing indexConfig settings for randomized testing.
-->
<indexConfig>
<!-- this sys property is not set by SolrTestCaseJ4 because we ideally want to use
the RandomMergePolicy in all tests - but some tests expect very specific
Merge behavior, so those tests can set it as needed.
-->
<mergePolicy class="${solr.tests.mergePolicy:org.apache.solr.util.RandomMergePolicy}" />
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
<maxBufferedDocs>${solr.tests.maxBufferedDocs}</maxBufferedDocs>
<maxIndexingThreads>${solr.tests.maxIndexingThreads}</maxIndexingThreads>
<ramBufferSizeMB>${solr.tests.ramBufferSizeMB}</ramBufferSizeMB>
<mergeScheduler class="${solr.tests.mergeScheduler}" />
<writeLockTimeout>1000</writeLockTimeout>
<commitLockTimeout>10000</commitLockTimeout>
<!-- this sys property is not set by SolrTestCaseJ4 because almost all tests should
use the single process lockType for speed - but tests that explicitly need
to vary the lockType canset it as needed.
-->
<lockType>${solr.tests.lockType:single}</lockType>
</indexConfig>
@@ -0,0 +1,567 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- This is a "kitchen sink" config file that tests can use.
When writting a new test, feel free to add *new* items (plugins,
config options, etc...) as long as they don't break any existing
tests. if you need to test something esoteric please add a new
"solrconfig-your-esoteric-purpose.xml" config file.
Note in particular that this test is used by MinimalSchemaTest so
Anything added to this file needs to work correctly even if there
is now uniqueKey or defaultSearch Field.
-->
<config>
<jmx />
<!-- Used to specify an alternate directory to hold all index data.
It defaults to "index" if not present, and should probably
not be changed if replication is in use. -->
<dataDir>${solr.data.dir:}</dataDir>
<!-- The DirectoryFactory to use for indexes.
solr.StandardDirectoryFactory, the default, is filesystem based.
solr.RAMDirectoryFactory is memory based and not persistent. -->
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}">
<double name="maxWriteMBPerSecDefault">1000000</double>
<double name="maxWriteMBPerSecFlush">2000000</double>
<double name="maxWriteMBPerSecMerge">3000000</double>
<double name="maxWriteMBPerSecRead">4000000</double>
</directoryFactory>
<schemaFactory class="ClassicIndexSchemaFactory"/>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
<updateHandler class="solr.DirectUpdateHandler2">
<!-- autocommit pending docs if certain criteria are met
<autoCommit>
<maxDocs>10000</maxDocs>
<maxTime>3600000</maxTime>
</autoCommit>
-->
<!-- represents a lower bound on the frequency that commits may
occur (in seconds). NOTE: not yet implemented
<commitIntervalLowerBound>0</commitIntervalLowerBound>
-->
<!-- The RunExecutableListener executes an external command.
exe - the name of the executable to run
dir - dir to use as the current working directory. default="."
wait - the calling thread waits until the executable returns. default="true"
args - the arguments to pass to the program. default=nothing
env - environment variables to set. default=nothing
-->
<!-- A postCommit event is fired after every commit
<listener event="postCommit" class="solr.RunExecutableListener">
<str name="exe">/var/opt/resin3/__PORT__/scripts/solr/snapshooter</str>
<str name="dir">/var/opt/resin3/__PORT__</str>
<bool name="wait">true</bool>
<arr name="args"> <str>arg1</str> <str>arg2</str> </arr>
<arr name="env"> <str>MYVAR=val1</str> </arr>
</listener>
-->
<!--
<updateLog enable="${enable.update.log:true}">
<str name="dir">${solr.ulog.dir:}</str>
</updateLog>
<commitWithin>
<softCommit>${solr.commitwithin.softcommit:true}</softCommit>
</commitWithin>
-->
</updateHandler>
<query>
<!-- Maximum number of clauses in a boolean query... can affect
range or wildcard queries that expand to big boolean
queries. An exception is thrown if exceeded.
-->
<maxBooleanClauses>1024</maxBooleanClauses>
<!-- Cache specification for Filters or DocSets - unordered set of *all* documents
that match a particular query.
-->
<filterCache
class="solr.search.FastLRUCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<queryResultCache
class="solr.search.LRUCache"
size="512"
initialSize="512"
autowarmCount="2"/>
<documentCache
class="solr.search.LRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<cache name="perSegFilter"
class="solr.search.LRUCache"
size="10"
initialSize="0"
autowarmCount="10" />
<!-- If true, stored fields that are not requested will be loaded lazily.
-->
<enableLazyFieldLoading>true</enableLazyFieldLoading>
<!--
<cache name="myUserCache"
class="solr.search.LRUCache"
size="4096"
initialSize="1024"
autowarmCount="1024"
regenerator="MyRegenerator"
/>
-->
<!--
<useFilterForSortedQuery>true</useFilterForSortedQuery>
-->
<queryResultWindowSize>10</queryResultWindowSize>
<!-- set maxSize artificially low to exercise both types of sets -->
<HashDocSet maxSize="3" loadFactor="0.75"/>
<!-- boolToFilterOptimizer converts boolean clauses with zero boost
into cached filters if the number of docs selected by the clause exceeds
the threshold (represented as a fraction of the total index)
-->
<boolTofilterOptimizer enabled="false" cacheSize="32" threshold=".05"/>
<!-- a newSearcher event is fired whenever a new searcher is being prepared
and there is a current searcher handling requests (aka registered). -->
<!-- QuerySenderListener takes an array of NamedList and executes a
local query request for each NamedList in sequence. -->
<!--
<listener event="newSearcher" class="solr.QuerySenderListener">
<arr name="queries">
<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>
<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>
</arr>
</listener>
-->
<!-- a firstSearcher event is fired whenever a new searcher is being
prepared but there is no current registered searcher to handle
requests or to gain prewarming data from. -->
<!--
<listener event="firstSearcher" class="solr.QuerySenderListener">
<arr name="queries">
<lst> <str name="q">fast_warm</str> <str name="start">0</str> <str name="rows">10</str> </lst>
</arr>
</listener>
-->
<listener event="firstSearcher" class="org.alfresco.solr.lifecycle.SolrCoreLoadListener" />
</query>
<queryResponseWriter name="xml" default="true"
class="solr.XMLResponseWriter" />
<requestHandler name="/replication" class="org.alfresco.solr.handler.AlfrescoReplicationHandler">
<lst name="slave">
<str name="masterUrl">{masterURL}</str>
<str name="pollInterval">00:00:02</str>
</lst>
</requestHandler>
<!-- An alternate set representation that uses an integer hash to store filters (sets of docids).
If the set cardinality <= maxSize elements, then HashDocSet will be used instead of the bitset
based HashBitset. -->
<!-- requestHandler plugins... incoming queries will be dispatched to the
correct handler based on the 'qt' param matching the
name of registered handlers.
The "standard" request handler is the default and will be used if qt
is not specified in the request.
-->
<requestHandler name="standard" class="solr.StandardRequestHandler">
<bool name="httpCaching">true</bool>
</requestHandler>
<requestHandler name="/get" class="solr.RealTimeGetHandler">
<lst name="defaults">
<str name="omitHeader">true</str>
</lst>
</requestHandler>
<requestHandler name="dismax" class="solr.SearchHandler" >
<lst name="defaults">
<str name="defType">dismax</str>
<str name="q.alt">*:*</str>
<float name="tie">0.01</float>
<str name="qf">
text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0
</str>
<str name="pf">
text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5
</str>
<str name="bf">
ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3
</str>
<str name="mm">
3&lt;-1 5&lt;-2 6&lt;90%
</str>
<int name="ps">100</int>
</lst>
</requestHandler>
<!-- test query parameter defaults -->
<requestHandler name="defaults" class="solr.StandardRequestHandler">
<lst name="defaults">
<int name="rows">4</int>
<bool name="hl">true</bool>
<str name="hl.fl">text,name,subject,title,whitetok</str>
</lst>
</requestHandler>
<!-- test query parameter defaults -->
<requestHandler name="lazy" class="solr.StandardRequestHandler" startup="lazy">
<lst name="defaults">
<int name="rows">4</int>
<bool name="hl">true</bool>
<str name="hl.fl">text,name,subject,title,whitetok</str>
</lst>
</requestHandler>
<requestHandler name="/update" class="solr.UpdateRequestHandler" />
<searchComponent name="fingerprint" class="org.alfresco.solr.component.FingerPrintComponent"/>
<requestHandler name="/fingerprint" class="org.apache.solr.handler.component.AlfrescoSearchHandler" lazy="true" >
<arr name="components">
<str>fingerprint</str>
</arr>
</requestHandler>
<requestHandler name="/afts" class="org.apache.solr.handler.component.AlfrescoSearchHandler" lazy="true" >
<lst name="defaults">
<str name="defType">afts</str>
<str name="spellcheck">false</str>
<str name="spellcheck.extendedResults">false</str>
<str name="spellcheck.count">5</str>
<str name="spellcheck.alternativeTermCount">2</str>
<str name="spellcheck.maxResultsForSuggest">5</str>
<str name="spellcheck.collate">true</str>
<str name="spellcheck.collateExtendedResults">true</str>
<str name="spellcheck.maxCollationTries">5</str>
<str name="spellcheck.maxCollations">3</str>
<str name="carrot.title">mltext@m___t@{http://www.alfresco.org/model/content/1.0}title</str>
<str name="carrot.url">id</str>
<str name="carrot.snippet">content@s___t@{http://www.alfresco.org/model/content/1.0}content</str>
<bool name="carrot.produceSummary">true</bool>
<bool name="carrot.outputSubClusters">false</bool>
</lst>
<arr name="components">
<str>setLocale</str>
<str>rewriteFacetParameters</str>
<str>query</str>
<str>facet</str>
<str>facet_module</str>
<str>mlt</str>
<str>highlight</str>
<str>stats</str>
<str>debug</str>
<str>clearLocale</str>
<str>rewriteFacetCounts</str>
<str>spellcheck</str>
<str>spellcheckbackcompat</str>
<str>setProcessedDenies</str>
</arr>
<shardHandlerFactory class="org.apache.solr.handler.component.AlfrescoHttpShardHandlerFactory" />
</requestHandler>
<requestHandler name="/native" class="org.apache.solr.handler.component.AlfrescoSearchHandler" lazy="true" >
<!-- default values for query parameters can be specified, these
will be overridden by parameters in the request
-->
<lst name="defaults">
<str name="echoParams">explicit</str>
<int name="rows">10</int>
<str name="df">suggest</str>
</lst>
<arr name="components">
<str>setLocale</str>
<str>query</str>
<str>facet</str>
<str>mlt</str>
<str>highlight</str>
<str>stats</str>
<str>debug</str>
<str>clearLocale</str>
</arr>
<shardHandlerFactory class="org.apache.solr.handler.component.AlfrescoHttpShardHandlerFactory" />
</requestHandler>
<requestHandler name="/cmis" class="org.apache.solr.handler.component.AlfrescoSearchHandler" lazy="true" >
<lst name="defaults">
<str name="defType">cmis</str>
</lst>
<arr name="components">
<str>query</str>
<str>facet</str>
<str>mlt</str>
<str>highlight</str>
<str>stats</str>
<str>debug</str>
</arr>
<shardHandlerFactory class="org.apache.solr.handler.component.AlfrescoHttpShardHandlerFactory" />
</requestHandler>
<searchComponent name="termsComp" class="org.apache.solr.handler.component.TermsComponent"/>
<requestHandler name="/terms" class="org.apache.solr.handler.component.SearchHandler">
<arr name="components">
<str>termsComp</str>
</arr>
</requestHandler>
<requestHandler name="mltrh" class="org.apache.solr.handler.component.SearchHandler">
</requestHandler>
<searchComponent name="tvComponent" class="org.apache.solr.handler.component.TermVectorComponent"/>
<requestHandler name="tvrh" class="org.apache.solr.handler.component.SearchHandler">
<lst name="defaults">
</lst>
<arr name="last-components">
<str>tvComponent</str>
</arr>
</requestHandler>
<requestHandler name="/mlt" class="solr.MoreLikeThisHandler">
</requestHandler>
<searchComponent class="solr.HighlightComponent" name="highlight">
<highlighting class="org.apache.solr.handler.component.AlfrescoSolrHighlighter">
<!-- Configure the standard fragmenter -->
<fragmenter name="gap" class="org.apache.solr.highlight.GapFragmenter" default="true">
<lst name="defaults">
<int name="hl.fragsize">100</int>
</lst>
</fragmenter>
<fragmenter name="regex" class="org.apache.solr.highlight.RegexFragmenter">
<lst name="defaults">
<!-- slightly smaller fragsizes work better because of slop -->
<int name="hl.fragsize">70</int>
<!-- allow 50% slop on fragment sizes -->
<float name="hl.regex.slop">0.5</float>
<!-- a basic sentence pattern -->
<str name="hl.regex.pattern">[-\w ,/\n\&quot;&apos;]{20,200}</str>
</lst>
</fragmenter>
<!-- Configure the standard formatter -->
<formatter name="html" class="org.apache.solr.highlight.HtmlFormatter" default="true">
<lst name="defaults">
<str name="hl.simple.pre"><![CDATA[<em>]]></str>
<str name="hl.simple.post"><![CDATA[</em>]]></str>
</lst>
</formatter>
<!-- Configure the standard encoder -->
<encoder name="html" class="solr.highlight.HtmlEncoder"/>
<!-- Configure the standard fragListBuilder -->
<fragListBuilder name="simple" class="solr.highlight.SimpleFragListBuilder"/>
<!-- Configure the single fragListBuilder -->
<fragListBuilder name="single" class="solr.highlight.SingleFragListBuilder"/>
<!-- Configure the weighted fragListBuilder -->
<fragListBuilder name="weighted" default="true" class="solr.highlight.WeightedFragListBuilder"/>
<!-- default tag FragmentsBuilder -->
<fragmentsBuilder name="default" default="true" class="solr.highlight.ScoreOrderFragmentsBuilder">
<!--
<lst name="defaults">
<str name="hl.multiValuedSeparatorChar">/</str>
</lst>
-->
</fragmentsBuilder>
<!-- multi-colored tag FragmentsBuilder -->
<fragmentsBuilder name="colored" class="solr.highlight.ScoreOrderFragmentsBuilder">
<lst name="defaults">
<str name="hl.tag.pre"><![CDATA[
<b style="background:yellow">,<b style="background:lawgreen">,
<b style="background:aquamarine">,<b style="background:magenta">,
<b style="background:palegreen">,<b style="background:coral">,
<b style="background:wheat">,<b style="background:khaki">,
<b style="background:lime">,<b style="background:deepskyblue">]]></str>
<str name="hl.tag.post"><![CDATA[</b>]]></str>
</lst>
</fragmentsBuilder>
<boundaryScanner name="default" default="true" class="solr.highlight.SimpleBoundaryScanner">
<lst name="defaults">
<str name="hl.bs.maxScan">10</str>
<str name="hl.bs.chars">.,!? &#9;&#10;&#13;</str>
</lst>
</boundaryScanner>
<boundaryScanner name="breakIterator" class="solr.highlight.BreakIteratorBoundaryScanner">
<lst name="defaults">
<!-- type should be one of CHARACTER, WORD(default), LINE and SENTENCE -->
<str name="hl.bs.type">WORD</str>
<!-- language and country are used when constructing Locale object. -->
<!-- And the Locale object will be used when getting instance of BreakIterator -->
<str name="hl.bs.language">en</str>
<str name="hl.bs.country">US</str>
</lst>
</boundaryScanner>
</highlighting>
</searchComponent>
<!-- enable streaming for testing... -->
<requestDispatcher handleSelect="true" >
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048" />
<httpCaching lastModifiedFrom="openTime" etagSeed="Solr" never304="false">
<cacheControl>max-age=30, public</cacheControl>
</httpCaching>
</requestDispatcher>
<!-- Echo the request contents back to the client -->
<requestHandler name="/debug/dump" class="solr.DumpRequestHandler" >
<lst name="defaults">
<str name="echoParams">explicit</str>
<str name="echoHandler">true</str>
</lst>
</requestHandler>
<admin>
<defaultQuery>solr</defaultQuery>
<gettableFiles>solrconfig.xml schema.xml admin-extra.html</gettableFiles>
</admin>
<!-- test getting system property -->
<!--
<propTest attr1="${solr.test.sys.prop1}-$${literal}"
attr2="${non.existent.sys.prop:default-from-config}">prefix-${solr.test.sys.prop2}-suffix</propTest>
-->
<queryParser name="alfrescoReRank" class="org.alfresco.solr.query.AlfrescoReRankQParserPlugin"/>
<queryParser name="afts" class="org.alfresco.solr.query.AlfrescoFTSQParserPlugin"/>
<!--
<queryParser name="afts" class="org.alfresco.solr.query.AlfrescoFTSQParserPlugin">
<str name="rerankPhase">SINGLE_PASS</str>
</queryParser>
-->
<queryParser name="cmis" class="org.alfresco.solr.query.CmisQParserPlugin"/>
<xi:include href="solrconfig_insight.xml" xmlns:xi="http://www.w3.org/2001/XInclude">
<xi:fallback/>
</xi:include>
<queryParser name="mimetype" class="org.alfresco.solr.query.MimetypeGroupingQParserPlugin" >
<str name="mapping">conf/mime_types.csv</str>
</queryParser>
<queryParser name="contentSize" class="org.alfresco.solr.query.ContentSizeGroupingQParserPlugin" >
<int name="scale">1</int>
<int name="buckets">10</int>
</queryParser>
<searchComponent name="setLocale" class="org.alfresco.solr.component.SetLocaleComponent" />
<searchComponent name="clearLocale" class="org.alfresco.solr.component.ClearLocaleComponent" />
<!-- Spell Check
The spell check component can return a list of alternative spelling
suggestions.
http://wiki.apache.org/solr/SpellCheckComponent
-->
<searchComponent name="spellcheck" class="org.alfresco.solr.component.spellcheck.AlfrescoSpellCheckComponent">
<str name="queryAnalyzerFieldType">text_shingle</str>
<!-- Multiple "Spell Checkers" can be declared and used by this
component
-->
<!-- a spellchecker built from a field of the main index -->
<lst name="spellchecker">
<str name="name">default</str>
<str name="field">suggest</str>
<str name="classname">solr.DirectSolrSpellChecker</str>
<!-- the spellcheck distance measure used, the default is the internal levenshtein -->
<str name="distanceMeasure">internal</str>
<!-- minimum accuracy needed to be considered a valid spellcheck suggestion -->
<float name="accuracy">0.5</float>
<!-- the maximum #edits we consider when enumerating terms: can be 1 or 2 -->
<int name="maxEdits">2</int>
<!-- the minimum shared prefix when enumerating terms -->
<int name="minPrefix">1</int>
<!-- maximum number of inspections per result. -->
<int name="maxInspections">5</int>
<!-- minimum length of a query term to be considered for correction -->
<int name="minQueryLength">4</int>
<!-- maximum threshold of documents a query term can appear to be considered for correction -->
<float name="maxQueryFrequency">0.01</float>
<!-- uncomment this to require suggestions to occur in 1% of the documents
<float name="thresholdTokenFrequency">.01</float>
-->
</lst>
<!-- word break -->
<lst name="spellchecker">
<str name="name">wordbreak</str>
<str name="field">suggest</str>
<str name="classname">solr.WordBreakSolrSpellChecker</str>
<str name="combineWords">true</str>
<str name="breakWords">true</str>
<int name="maxChanges">10</int>
<int name="minBreakLength">5</int>
</lst>
</searchComponent>
<searchComponent name="spellcheckbackcompat" class="org.alfresco.solr.component.spellcheck.AlfrescoSpellCheckBackCompatComponent"/>
<searchComponent name="rewriteFacetParameters" class="org.alfresco.solr.component.RewriteFacetParametersComponent" />
<searchComponent name="rewriteFacetCounts" class="org.alfresco.solr.component.RewriteFacetCountsComponent" />
<searchComponent name="setProcessedDenies" class="org.alfresco.solr.component.SetProcessedDeniesComponent" />
<transformer name="cached" class="org.alfresco.solr.transformer.CachedDocTransformerFactory" >
</transformer>
</config>
@@ -0,0 +1,154 @@
#
# solrcore.properties - used in solrconfig.xml
#
enable.alfresco.tracking=false
#
#These are replaced by the admin handler
#
#data.dir.root=DATA_DIR
#data.dir.store=workspace/SpacesStore
#alfresco.stores=workspace://SpacesStore
#
# Properties loaded during alfresco tracking
#
alfresco.host=localhost
alfresco.port=8080
alfresco.port.ssl=8443
alfresco.baseUrl=/alfresco
alfresco.cron=0/2 * * * * ? *
#alfresco.index.transformContent=false
#alfresco.ignore.datatype.1=d:content
alfresco.lag=1000
alfresco.hole.retention=3600000
# alfresco.hole.check.after is not used yet
# It will reduce the hole checking load
alfresco.hole.check.after=300000
alfresco.batch.count=1000
# encryption
# none, https
alfresco.secureComms=none
# ssl
alfresco.encryption.ssl.keystore.type=JCEKS
alfresco.encryption.ssl.keystore.provider=
alfresco.encryption.ssl.keystore.location=ssl.repo.client.keystore
alfresco.encryption.ssl.keystore.passwordFileLocation=ssl-keystore-passwords.properties
alfresco.encryption.ssl.truststore.type=JCEKS
alfresco.encryption.ssl.truststore.provider=
alfresco.encryption.ssl.truststore.location=ssl.repo.client.truststore
alfresco.encryption.ssl.truststore.passwordFileLocation=ssl-truststore-passwords.properties
# Tracking
alfresco.corePoolSize=1
alfresco.maximumPoolSize=-1
alfresco.keepAliveTime=120
alfresco.threadPriority=5
alfresco.threadDaemon=true
alfresco.workQueueSize=-1
# HTTP Client
alfresco.maxTotalConnections=200
alfresco.maxHostConnections=200
alfresco.socketTimeout=360000
# SOLR caching
solr.filterCache.size=256
solr.filterCache.initialSize=128
solr.queryResultCache.size=1024
solr.queryResultCache.initialSize=1024
solr.documentCache.size=1024
solr.documentCache.initialSize=1024
solr.queryResultMaxDocsCached=2048
solr.authorityCache.size=128
solr.authorityCache.initialSize=64
solr.pathCache.size=256
solr.pathCache.initialSize=128
solr.ownerCache.size=128
solr.ownerCache.initialSize=64
solr.readerCache.size=128
solr.readerCache.initialSize=64
solr.deniedCache.size=128
solr.deniedCache.initialSize=64
# SOLR
solr.maxBooleanClauses=10000
# Batch fetch
alfresco.transactionDocsBatchSize=100
alfresco.nodeBatchSize=10
alfresco.changeSetAclsBatchSize=100
alfresco.aclBatchSize=10
alfresco.contentReadBatchSize=4000
alfresco.contentUpdateBatchSize=1000
# Warming
solr.filterCache.autowarmCount=32
solr.authorityCache.autowarmCount=4
solr.pathCache.autowarmCount=32
solr.deniedCache.autowarmCount=0
solr.readerCache.autowarmCount=0
solr.ownerCache.autowarmCount=0
solr.queryResultCache.autowarmCount=4
solr.documentCache.autowarmCount=512
solr.queryResultWindowSize=512
#
# TODO
#
# cross language support
# locale expansion
# logging check report ....
#
#
alfresco.commitInterval=1000
alfresco.newSearcherInterval=2000
alfresco.doPermissionChecks=true
#
# Metadata pulling control
#
alfresco.metadata.skipDescendantDocsForSpecificTypes=false
alfresco.metadata.ignore.datatype.0=cm:person
alfresco.metadata.ignore.datatype.1=app:configurations
alfresco.metadata.skipDescendantDocsForSpecificAspects=false
#alfresco.metadata.ignore.aspect.0=
#
# Suggestions
#
solr.suggester.enabled=false
# -1 to disable suggester build throttling
solr.suggester.minSecsBetweenBuilds=3600
#
# Limit the maximum text size of transformed content sent to the index - in bytes
#
alfresco.contentStreamLimit=10000000
#Sharding default values
shard.instance=1
shard.count=0
shard.method=DB_ID
@@ -0,0 +1,50 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
a
an
and
are
as
at
be
but
by
for
if
in
into
is
it
no
not
of
on
or
s
such
t
that
the
their
then
there
these
they
this
to
was
will
with
@@ -0,0 +1,35 @@
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#-----------------------------------------------------------------------
#some test synonym mappings unlikely to appear in real input text
aaa => aaaa
bbb => bbbb1 bbbb2
ccc => cccc1,cccc2
a\=>a => b\=>b
a\,a => b\,b
fooaaa,baraaa,bazaaa
# Some synonym groups specific to this example
GB,gib,gigabyte,gigabytes
MB,mib,megabyte,megabytes
Television, Televisions, TV, TVs
#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming
#after us won't split it into two words.
# Synonym mappings can be used for spelling correction too
pixima => pixma
# Test synonyms
quick,fast,rapid,speedy
brown fox jumped,leaping reynard,springer
lazy,bone idle
@@ -15,7 +15,7 @@
<httpCaching never304="true" />
</requestDispatcher>
<requestHandler name="/replication" class="io.sease.labs.solr.handler.ReplicationHandler">
<requestHandler name="/replication" class="solr.ReplicationHandler">
<lst name="master">
<str name="replicateAfter">commit</str>
<str name="confFiles">schema.xml</str>
@@ -15,7 +15,7 @@
<httpCaching never304="true" />
</requestDispatcher>
<requestHandler name="/replication" class="io.sease.labs.solr.handler.ReplicationHandler">
<requestHandler name="/replication" class="solr.ReplicationHandler">
<lst name="master">
<str name="enable">false</str>
<str name="replicateAfter">commit</str>
@@ -15,7 +15,7 @@
<httpCaching never304="true" />
</requestDispatcher>
<requestHandler name="/replication" class="io.sease.labs.solr.handler.ReplicationHandler">
<requestHandler name="/replication" class="solr.ReplicationHandler">
<lst name="master">
<str name="replicateAfter">commit</str>
<str name="confFiles">schema.xml</str>
@@ -23,7 +23,7 @@
<properties>
<dependency.alfresco-data-model.version>8.53</dependency.alfresco-data-model.version>
<dependency.jackson.version>2.10.0</dependency.jackson.version>
<dependency.jackson.version>2.10.1</dependency.jackson.version>
</properties>
<dependencies>
@@ -54,9 +54,9 @@ mybatis-spring-1.2.5.jar http://www.mybatis.org/
chemistry-opencmis-server-support-1.0.0.jar http://chemistry.apache.org/
chemistry-opencmis-server-bindings-1.0.0.jar http://chemistry.apache.org/
quartz-2.3.1.jar http://quartz-scheduler.org/
jackson-core-2.10.0.jar https://github.com/FasterXML/jackson
jackson-annotations-2.10.0.jar https://github.com/FasterXML/jackson
jackson-databind-2.10.0.jar https://github.com/FasterXML/jackson
jackson-core-2.10.1.jar https://github.com/FasterXML/jackson
jackson-annotations-2.10.1.jar https://github.com/FasterXML/jackson
jackson-databind-2.10.1.jar https://github.com/FasterXML/jackson
commons-httpclient-3.1-HTTPCLIENT-1265.jar http://jakarta.apache.org/commons/
spring-aop-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/
spring-beans-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/