mirror of
https://github.com/Alfresco/SearchServices.git
synced 2026-09-16 18:12:56 +00:00
Merge master into feature/SEARCH-1745_WrapNPEForShardKey.
This commit is contained in:
@@ -16,29 +16,43 @@ Following templates are provided.
|
||||
$ tree generators/app/templates/
|
||||
generators/app/templates/
|
||||
├── 6.1
|
||||
│ ├── .env
|
||||
│ ├── docker-compose-ce.yml
|
||||
│ └── docker-compose-ee.yml
|
||||
├── 6.2
|
||||
│ ├── .env
|
||||
│ ├── docker-compose-ce.yml
|
||||
│ └── docker-compose-ee.yml
|
||||
├── images
|
||||
│ ├── alfresco
|
||||
│ │ ├── Dockerfile
|
||||
│ │ └── model
|
||||
│ │ ├── empty
|
||||
│ │ ├── sharding-content-model-context.xml
|
||||
│ │ └── sharding-content-model.xml
|
||||
│ ├── search
|
||||
│ │ └── Dockerfile
|
||||
│ ├── share
|
||||
│ │ ├── Dockerfile
|
||||
│ │ └── model
|
||||
│ │ ├── empty.xml
|
||||
│ │ └── sharding-share-config-custom.xml
|
||||
│ └── zeppelin
|
||||
│ └── Dockerfile
|
||||
└── keystores
|
||||
├── alfresco
|
||||
├── client
|
||||
├── solr
|
||||
└── zeppelin
|
||||
```
|
||||
|
||||
* `.env` includes default values for Docker Compose environment variables
|
||||
* `docker-compose-ce.yml` is the base Docker Compose Template for Alfresco Community deployment (for ACS 6.2 and ACS 6.1)
|
||||
* `docker-compose-ee.yml` is the base Docker Compose Template for Alfresco Enterprise deployment (for ACS 6.2 and ACS 6.1)
|
||||
* `alfresco` includes a Dockerfile template to start Alfresco Repository
|
||||
* `model` includes a default content model for Sharding Explicit Routing
|
||||
* `model` includes a default content model (Sharding Explicit Routing or empty). This `empty` file is required for Dockerfile to work, so it should not be deleted.
|
||||
* `share` includes a Dockerfile template to start Share Web Application
|
||||
* `model` includes a default forms model (Sharding Explicit Routing or empty)
|
||||
* `search` includes a Dockerfile template to start Search Services and Insight Engine
|
||||
* `zeppelin` includes a Dockerfile template to start Zeppelin with SSL
|
||||
* `keystores` includes every truststore and keystore required for SSL configuration
|
||||
@@ -62,14 +76,24 @@ $ yo alfresco-docker-compose
|
||||
|
||||
## ACS Version
|
||||
|
||||
Currently supported ACS Version are `6.2` and `6.1`
|
||||
Currently supported ACS Versions are `6.2` and `6.1`
|
||||
|
||||
This is the first choice to be selected when the generator is executed.
|
||||
|
||||
```
|
||||
? Which ACS version do you want to use?
|
||||
6.1
|
||||
❯ 6.2
|
||||
```
|
||||
## AGS Version
|
||||
|
||||
Currently supported AGS Version is `latest` (based in ACS 6.1).
|
||||
|
||||
If you chose ACS 6.1, a prompt will allow you to use AGS.
|
||||
|
||||
```
|
||||
? Would you like to use AGS? Yes
|
||||
```
|
||||
|
||||
## Community
|
||||
|
||||
@@ -122,22 +146,45 @@ Shard property is `shard:shardId`, belonging to a custom model deployed in the t
|
||||
**EXPLICIT_ID** and **EXPLICIT_ID_FALLBACK_LRIS**
|
||||
Custom content model is deployed to provide a property, named `shard:shardId`, holding the Shard Number (0, 1) where the content is indexed.
|
||||
|
||||
This default configuration can be changed in the generated `docker-compose.yml` template.
|
||||
|
||||
|
||||
## Deploying custom content models and forms
|
||||
|
||||
Custom content models and Share Form configurations can be added to deployment folders.
|
||||
|
||||
**Content models**
|
||||
|
||||
Custom content models can be copied to Repository deployment folder by using bootstrap approach.
|
||||
|
||||
Following XML files must be created in `alfresco/model/` folder in the Docker Compose template generated:
|
||||
|
||||
* `content-model.xml` including an XML Alfresco Content Model file. Sample model is available in [images/alfresco/model/sharding-content-model.xml](generators/app/templates/images/alfresco/model/sharding-content-model.xml)
|
||||
* `content-model-context.xml` including an XML Spring Bean file with the `dictionaryBootstrap` bean. Sample Spring Bean declaration is available in [images/alfresco/model/sharding-content-model-context.xml](generators/app/templates/images/alfresco/model/sharding-content-model-context.xml)
|
||||
|
||||
If *Sharding* is selected, these files will be available in deployment folder.
|
||||
|
||||
**Share forms**
|
||||
|
||||
Custom content forms can be added to Share configuration by modifying `share/model/share-config-custom-dev.xml` file in the Docker Compose template generated.
|
||||
|
||||
Sample configuration is available in [images/share/model/sharding-share-config-custom.xml](generators/app/templates/images/share/model/sharding-share-config-custom.xml)
|
||||
|
||||
If *Sharding* is selected, a default `share-config-custom-dev.xml` file with required forms configuration for Sharding custom model will be available in deployment folder. Add your configuration to this file.
|
||||
|
||||
## Configuration catalog
|
||||
|
||||
| Version | Comms | Replication | Sharding | Explicit | Insight | Zeppelin |
|
||||
| - | - | - | - | - | - | - |
|
||||
| community | http | - | - | - | x | x |
|
||||
| community | http | true | x | x | x | x |
|
||||
| community | http | false | true | false | x | x |
|
||||
| community | http | false | true | true | x | x |
|
||||
| community | https | x | - | - | x | x |
|
||||
| enterprise | http | - | - | - | (*) | (*) |
|
||||
| enterprise | http | true | x | x | (*) | (*) |
|
||||
| enterprise | http | false | true | false | (*) | (*) |
|
||||
| enterprise | http | false | true | true | (*) | (*) |
|
||||
| enterprise | https | x | - | - | (*) | (*) |
|
||||
| enterprise | https | x | true | false | (*) | (*) |
|
||||
| enterprise | https | x | true | true | (*) | (*) |
|
||||
| Version | Comms | Replication | Sharding | Insight | Zeppelin |
|
||||
| - | - | - | - | - | - |
|
||||
| community | http | - | x | x | x |
|
||||
| community | http | true | x | x | x |
|
||||
| community | https | x | x | x | x |
|
||||
| enterprise | http | - | - | (*) | (*) |
|
||||
| enterprise | http | true | false | (*) | (*) |
|
||||
| enterprise | http | false | true | (*) | (*) |
|
||||
| enterprise | https | x | - | (*) | (*) |
|
||||
| enterprise | https | x | true | (*) | (*) |
|
||||
| enterprise | https | x | true | (*) | (*) |
|
||||
|
||||
Both `community` and `enterprise` ACS deployments can be used with the same options, but `enterprise` may also use Insight Engine (replacing Search Services) and Insight Zeppelin services.
|
||||
|
||||
@@ -152,6 +199,7 @@ $ yo alfresco-docker-compose --acsVersion=6.2 --alfrescoVersion=community --http
|
||||
**Parameter names reference**
|
||||
|
||||
`--acsVersion`: default 6.2, but could be set to 6.1
|
||||
`--ags:`: only available when acsVersion=6.1
|
||||
`--alfrescoVersion`: community or enterprise
|
||||
`--httpMode`: http or https
|
||||
`--clustering`: true or false
|
||||
@@ -180,7 +228,7 @@ $ docker-compose down
|
||||
|
||||
http://localhost:8080/share
|
||||
|
||||
http://localhost:8082/alfresco
|
||||
http://localhost:8080/alfresco
|
||||
|
||||
http://localhost:8083/solr
|
||||
|
||||
@@ -193,7 +241,7 @@ http://localhost:8084/solr
|
||||
|
||||
http://localhost:8080/share
|
||||
|
||||
http://localhost:8082/alfresco
|
||||
http://localhost:8080/alfresco
|
||||
|
||||
https://localhost:8443/alfresco
|
||||
|
||||
|
||||
@@ -28,6 +28,13 @@ module.exports = class extends Generator {
|
||||
choices: [ "6.1", "6.2" ],
|
||||
default: '6.2'
|
||||
},
|
||||
{
|
||||
whenFunction: response => response.acsVersion == '6.1',
|
||||
type: 'confirm',
|
||||
name: 'ags',
|
||||
message: 'Would you like to use AGS?',
|
||||
default: false
|
||||
},
|
||||
{
|
||||
type: 'list',
|
||||
name: 'alfrescoVersion',
|
||||
@@ -122,9 +129,13 @@ module.exports = class extends Generator {
|
||||
}
|
||||
|
||||
// Docker Compose environment variables values
|
||||
this.fs.copy(
|
||||
this.fs.copyTpl(
|
||||
this.templatePath(dockerComposeTemplateDirectory + '/.env'),
|
||||
this.destinationPath('.env')
|
||||
this.destinationPath('.env'),
|
||||
{
|
||||
acs: (this.props.ags ? "false" : "true"),
|
||||
ags: (this.props.ags ? "true" : "false")
|
||||
}
|
||||
)
|
||||
|
||||
// Base Docker Compose Template
|
||||
@@ -136,8 +147,26 @@ module.exports = class extends Generator {
|
||||
// Repository Docker Image name
|
||||
const acsImageName =
|
||||
(this.props.alfrescoVersion == 'community' ?
|
||||
'alfresco/alfresco-content-repository-community' :
|
||||
'alfresco/alfresco-content-repository');
|
||||
(this.props.ags ?
|
||||
'alfresco/alfresco-governance-repository-community' :
|
||||
'alfresco/alfresco-content-repository-community') :
|
||||
(this.props.ags ?
|
||||
'quay.io/alfresco/alfresco-governance-repository-enterprise':
|
||||
'alfresco/alfresco-content-repository'
|
||||
)
|
||||
);
|
||||
|
||||
// Share Docker Image name
|
||||
const shareImageName =
|
||||
(this.props.alfrescoVersion == 'community' ?
|
||||
(this.props.ags ?
|
||||
'alfresco/alfresco-governance-share-community' :
|
||||
'alfresco/alfresco-share') :
|
||||
(this.props.ags ?
|
||||
'quay.io/alfresco/alfresco-governance-share-enterprise':
|
||||
'alfresco/alfresco-share'
|
||||
)
|
||||
);
|
||||
|
||||
// Search Docker Image
|
||||
const searchImageName =
|
||||
@@ -179,10 +208,37 @@ module.exports = class extends Generator {
|
||||
sharding: (this.props.sharding ? "true" : "false")
|
||||
}
|
||||
);
|
||||
// Copy Sharding Content Model or an empty file to allow model deployments
|
||||
if (this.props.sharding) {
|
||||
this.fs.copy(
|
||||
this.templatePath(imagesDirectory + '/alfresco/model'),
|
||||
this.destinationPath('alfresco/model')
|
||||
this.templatePath(imagesDirectory + '/alfresco/model/*.xml'),
|
||||
this.destinationPath('alfresco/model/')
|
||||
)
|
||||
} else {
|
||||
this.fs.copy(
|
||||
this.templatePath(imagesDirectory + '/alfresco/model/empty'),
|
||||
this.destinationPath('alfresco/model/empty')
|
||||
)
|
||||
}
|
||||
|
||||
// Copy Docker Image for Share applying configuration
|
||||
this.fs.copyTpl(
|
||||
this.templatePath(imagesDirectory + '/share/Dockerfile'),
|
||||
this.destinationPath('share/Dockerfile'),
|
||||
{
|
||||
shareImage: shareImageName
|
||||
}
|
||||
);
|
||||
// Copy Sharding Content Forms or an empty file to allow forms deployments
|
||||
if (this.props.sharding) {
|
||||
this.fs.copy(
|
||||
this.templatePath(imagesDirectory + '/share/model/sharding-share-config-custom.xml'),
|
||||
this.destinationPath('share/model/share-config-custom-dev.xml')
|
||||
)
|
||||
} else {
|
||||
this.fs.copy(
|
||||
this.templatePath(imagesDirectory + '/share/model/empty.xml'),
|
||||
this.destinationPath('share/model/share-config-custom-dev.xml')
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
<% if (acs == 'true') { %>
|
||||
ALFRESCO_TAG=6.1.0.5
|
||||
ALFRESCO_CE_TAG=6.1.2-ga
|
||||
SHARE_TAG=6.1.0
|
||||
ALFRESCO_CE_TAG=6.1.2-ga
|
||||
SHARE_TAG=6.1.0 <% } %>
|
||||
<% if (ags == 'true') { %>
|
||||
ALFRESCO_TAG=latest
|
||||
ALFRESCO_CE_TAG=latest
|
||||
SHARE_TAG=latest <% } %>
|
||||
POSTGRES_TAG=10.1
|
||||
TRANSFORM_ROUTER_TAG=1.0.1
|
||||
PDF_RENDERER_TAG=2.0.10
|
||||
@@ -11,5 +16,6 @@ SHARED_FILE_STORE_TAG=0.5.3
|
||||
ACTIVE_MQ_TAG=5.15.8
|
||||
DIGITAL_WORKSPACE_TAG=1.1.0
|
||||
ACS_NGINX_TAG=3.0.0
|
||||
ACS_COMMUNITY_NGINX_TAG=1.0.0
|
||||
SEARCH_TAG=latest
|
||||
ZEPPELIN_TAG=latest
|
||||
|
||||
+16
-6
@@ -32,10 +32,8 @@ services:
|
||||
-Ddeployment.method=DOCKER_COMPOSE
|
||||
-Dcsrf.filter.enabled=false
|
||||
-Dalfresco.restApi.basicAuthScheme=true
|
||||
-Xms1500m -Xmx1500m
|
||||
"
|
||||
" <% if (httpMode == 'https') { %>
|
||||
ports:
|
||||
- 8082:8080 <% if (httpMode == 'https') { %>
|
||||
- 8443:8443
|
||||
volumes:
|
||||
- ./keystores/alfresco:/usr/local/tomcat/alf_data/keystore <% } %>
|
||||
@@ -124,7 +122,10 @@ services:
|
||||
<% } %>
|
||||
|
||||
share:
|
||||
image: alfresco/alfresco-share:${SHARE_TAG}
|
||||
build:
|
||||
context: ./share
|
||||
args:
|
||||
SHARE_TAG: ${SHARE_TAG}
|
||||
mem_limit: 1g
|
||||
environment:
|
||||
REPO_HOST: "alfresco"
|
||||
@@ -137,8 +138,6 @@ services:
|
||||
-Dalfresco.context=alfresco
|
||||
-Dalfresco.protocol=http
|
||||
"
|
||||
ports:
|
||||
- 8080:8080
|
||||
|
||||
postgres:
|
||||
image: postgres:${POSTGRES_TAG}
|
||||
@@ -159,3 +158,14 @@ services:
|
||||
- 5672:5672 # AMQP
|
||||
- 61616:61616 # OpenWire
|
||||
- 61613:61613 # STOMP
|
||||
|
||||
proxy:
|
||||
image: alfresco/acs-community-ngnix:${ACS_COMMUNITY_NGINX_TAG}
|
||||
mem_limit: 128m
|
||||
depends_on:
|
||||
- alfresco
|
||||
ports:
|
||||
- 8080:8080
|
||||
links:
|
||||
- alfresco
|
||||
- share
|
||||
+4
-1
@@ -264,7 +264,10 @@ services:
|
||||
- shared-file-store-volume:/tmp/Alfresco/sfs
|
||||
|
||||
share:
|
||||
image: alfresco/alfresco-share:${SHARE_TAG}
|
||||
build:
|
||||
context: ./share
|
||||
args:
|
||||
SHARE_TAG: ${SHARE_TAG}
|
||||
mem_limit: 1g
|
||||
environment:
|
||||
REPO_HOST: "alfresco"
|
||||
|
||||
+4
-1
@@ -178,7 +178,10 @@ services:
|
||||
- 8094:8090
|
||||
|
||||
share:
|
||||
image: alfresco/alfresco-share:${SHARE_TAG}
|
||||
build:
|
||||
context: ./share
|
||||
args:
|
||||
SHARE_TAG: ${SHARE_TAG}
|
||||
mem_limit: 1g
|
||||
environment:
|
||||
REPO_HOST: "alfresco"
|
||||
|
||||
+4
-1
@@ -287,7 +287,10 @@ services:
|
||||
- shared-file-store-volume:/tmp/Alfresco/sfs
|
||||
|
||||
share:
|
||||
image: alfresco/alfresco-share:${SHARE_TAG}
|
||||
build:
|
||||
context: ./share
|
||||
args:
|
||||
SHARE_TAG: ${SHARE_TAG}
|
||||
mem_limit: 1g
|
||||
environment:
|
||||
REPO_HOST: "alfresco"
|
||||
|
||||
+2
-4
@@ -48,7 +48,5 @@ RUN if [ "$SOLR_COMMS" == "https" ] ; then \
|
||||
<\/Connector>/g" ${TOMCAT_DIR}/conf/server.xml; \
|
||||
fi
|
||||
|
||||
<% if (sharding == 'true') { %>
|
||||
# Explicit Routing Sharding content model (useless when not using this feature)
|
||||
COPY model/sharding-content-model*.xml $TOMCAT_DIR/shared/classes/alfresco/extension/
|
||||
<% } %>
|
||||
# Copy custom content model to deployment folder
|
||||
COPY model/* $TOMCAT_DIR/shared/classes/alfresco/extension/
|
||||
|
||||
+2
@@ -13,9 +13,11 @@
|
||||
|
||||
<aspects>
|
||||
<aspect name="shard:sharding">
|
||||
<title>Sharding</title>
|
||||
<properties>
|
||||
<!-- Shard number to index this content -->
|
||||
<property name="shard:shardId">
|
||||
<title>Shard Id</title>
|
||||
<type>d:text</type>
|
||||
</property>
|
||||
</properties>
|
||||
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
ARG SHARE_TAG
|
||||
FROM <%=shareImage%>:${SHARE_TAG}
|
||||
|
||||
ARG TOMCAT_DIR=/usr/local/tomcat
|
||||
|
||||
# Copy custom content forms to deployment folder
|
||||
COPY model/* $TOMCAT_DIR/shared/classes/alfresco/web-extension/
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
<alfresco-config>
|
||||
|
||||
<!-- Add custom configuration -->
|
||||
|
||||
</alfresco-config>
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
<alfresco-config>
|
||||
|
||||
<!-- Add custom configuration -->
|
||||
|
||||
<config evaluator="string-compare" condition="DocumentLibrary">
|
||||
<aspects>
|
||||
<visible>
|
||||
<aspect name="shard:sharding"/>
|
||||
</visible>
|
||||
<addable>
|
||||
</addable>
|
||||
<removeable>
|
||||
</removeable>
|
||||
</aspects>
|
||||
</config>
|
||||
|
||||
<config evaluator="aspect" condition="shard:sharding">
|
||||
<forms>
|
||||
<form>
|
||||
<field-visibility>
|
||||
<show id="shard:shardId" />
|
||||
</field-visibility>
|
||||
</form>
|
||||
</forms>
|
||||
</config>
|
||||
|
||||
</alfresco-config>
|
||||
+4
-3
@@ -11,9 +11,9 @@
|
||||
<name>Search Analytics E2E Tests</name>
|
||||
<description>Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share</description>
|
||||
<properties>
|
||||
<tas.rest.api.version>6.0.0.3</tas.rest.api.version>
|
||||
<tas.cmis.api.version>6.0.0.1</tas.cmis.api.version>
|
||||
<tas.utility.version>2.0.12</tas.utility.version>
|
||||
<tas.rest.api.version>6.0.1.2</tas.rest.api.version>
|
||||
<tas.cmis.api.version>6.0.0.4</tas.cmis.api.version>
|
||||
<tas.utility.version>3.0.11</tas.utility.version>
|
||||
<rm.version>2.6.0</rm.version>
|
||||
<suiteXmlFile>src/test/resources/SearchSuite.xml</suiteXmlFile>
|
||||
<test.exclude></test.exclude>
|
||||
@@ -45,6 +45,7 @@
|
||||
</suiteXmlFiles>
|
||||
<excludedGroups>${test.exclude}</excludedGroups>
|
||||
<groups>${test.include}</groups>
|
||||
<redirectTestOutputToFile>true</redirectTestOutputToFile>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
@@ -1,33 +1,43 @@
|
||||
/*
|
||||
* Copyright (C) 2018 Alfresco Software Limited.
|
||||
*
|
||||
* This file is part of Alfresco
|
||||
*
|
||||
* Alfresco is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Alfresco is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.alfresco.search;
|
||||
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target({ ElementType.METHOD })
|
||||
public @interface TestGroup
|
||||
public class TestGroup
|
||||
{
|
||||
// Used for TestRail test annotation
|
||||
String SEARCH = "search";
|
||||
String REST_API = "rest-api";
|
||||
// Used for TestRail test annotation
|
||||
public static final String SEARCH = "search";
|
||||
public static final String REST_API = "rest-api";
|
||||
|
||||
String PREUPGRADE = "pre-upgrade";
|
||||
String POSTUPGRADE = "post-upgrade";
|
||||
public static final String PREUPGRADE = "pre-upgrade";
|
||||
public static final String POSTUPGRADE = "post-upgrade";
|
||||
|
||||
String ASS_MASTER_SLAVE = "ASS_Master_Slave"; // Alfresco Search Services using master slave configurations
|
||||
String ASS_MASTER ="ASS_Master"; // Alfresco search services using master/stand alone mode
|
||||
String EXPLICIT_SHARDING ="Explicit_Sharding"; // Alfresco search services using sharded environment and explicit routing
|
||||
public static final String ASS_MASTER_SLAVE = "ASS_Master_Slave"; // Alfresco Search Services using master slave configurations
|
||||
public static final String ASS_MASTER ="ASS_Master"; // Alfresco search services using master/stand alone mode
|
||||
public static final String EXPLICIT_SHARDING ="Explicit_Sharding"; // Alfresco search services using sharded environment and explicit routing
|
||||
|
||||
String NOT_INSIGHT_ENGINE = "Not_InsightEngine"; // When Alfresco Insight Engine 1.0 isn't running
|
||||
public static final String NOT_INSIGHT_ENGINE = "Not_InsightEngine"; // When Alfresco Insight Engine 1.0 isn't running
|
||||
|
||||
String ACS_52n = "ACS_52n"; // Alfresco Content Services 5.2.n
|
||||
String ACS_60n = "ACS_60n"; // Alfresco Content Services 6.0 or above
|
||||
String ACS_61n = "ACS_61n"; // Alfresco Content Services 6.1 or above
|
||||
String ACS_611n = "ACS_611n"; // Alfresco Content Services 6.1.1 or above
|
||||
String ACS_62n = "ACS_62n"; // Alfresco Content Services 6.2 or above
|
||||
|
||||
String NOT_BAMBOO = "Not_Bamboo"; // The does not run on bamboo
|
||||
public static final String ACS_52n = "ACS_52n"; // Alfresco Content Services 5.2.n
|
||||
public static final String ACS_60n = "ACS_60n"; // Alfresco Content Services 6.0 or above
|
||||
public static final String ACS_61n = "ACS_61n"; // Alfresco Content Services 6.1 or above
|
||||
public static final String ACS_611n = "ACS_611n"; // Alfresco Content Services 6.1.1 or above
|
||||
public static final String ACS_62n = "ACS_62n"; // Alfresco Content Services 6.2 or above
|
||||
|
||||
public static final String NOT_BAMBOO = "Not_Bamboo"; // The does not run on bamboo
|
||||
}
|
||||
|
||||
+33
-57
@@ -47,9 +47,10 @@ import static lombok.AccessLevel.PROTECTED;
|
||||
@ContextConfiguration("classpath:alfresco-search-e2e-context.xml")
|
||||
public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringContextTests
|
||||
{
|
||||
public static final int SEARCH_MAX_ATTEMPS = 6;
|
||||
/** The number of retries that a query will be tried before giving up. */
|
||||
private static final int SEARCH_MAX_ATTEMPTS = 6;
|
||||
|
||||
private static Logger LOG = LogFactory.getLogger();
|
||||
private static final Logger LOGGER = LogFactory.getLogger();
|
||||
|
||||
@Autowired
|
||||
protected RestProperties restProperties;
|
||||
@@ -85,8 +86,6 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
|
||||
protected static String unique_searchString;
|
||||
|
||||
public static final String NODE_PREFIX = "workspace/SpacesStore/";
|
||||
|
||||
@BeforeSuite(alwaysRun = true)
|
||||
public void beforeSuite() throws Exception
|
||||
{
|
||||
@@ -97,7 +96,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
}
|
||||
|
||||
@BeforeClass(alwaysRun = true)
|
||||
public void setup() throws Exception
|
||||
public void setup()
|
||||
{
|
||||
serverHealth.assertServerIsOnline();
|
||||
|
||||
@@ -127,7 +126,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LOG.warn("Error Loading Custom Model", e);
|
||||
LOGGER.warn("Error Loading Custom Model", e);
|
||||
}
|
||||
}
|
||||
return modelDeployed;
|
||||
@@ -135,8 +134,6 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
|
||||
public boolean deactivateCustomModel(String fileName)
|
||||
{
|
||||
Boolean modelDeactivated = false;
|
||||
|
||||
try
|
||||
{
|
||||
FileModel customModel = getCustomModel(fileName);
|
||||
@@ -144,17 +141,16 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
// Deactivate the model if found
|
||||
if (customModel != null)
|
||||
{
|
||||
|
||||
cmisApi.authenticateUser(dataUser.getAdminUser()).usingResource(customModel).updateProperty("cm:modelActive", false);
|
||||
|
||||
modelDeactivated = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LOG.warn("Error Deactivating Custom Model", e);
|
||||
LOGGER.warn("Error Deactivating Custom Model", e);
|
||||
}
|
||||
return modelDeactivated;
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean deleteCustomModel(String fileName)
|
||||
@@ -176,12 +172,12 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG.error("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName);
|
||||
LOGGER.error("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LOG.error("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName);
|
||||
LOGGER.error("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName);
|
||||
}
|
||||
|
||||
return modelDeleted;
|
||||
@@ -206,17 +202,17 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
customModel.setNodeRef(modelInRepo.getId());
|
||||
customModel.setNodeRef(customModel.getNodeRefWithoutVersion());
|
||||
customModel.setCmisLocation(String.format("/Data Dictionary/Models/%s", fileName));
|
||||
LOG.info("Custom Model file: " + customModel.getCmisLocation());
|
||||
LOGGER.info("Custom Model file: " + customModel.getCmisLocation());
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG.info("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName);
|
||||
LOGGER.info("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LOG.warn("Error Getting Custom Model: " + fileName, e);
|
||||
LOGGER.warn("Error Getting Custom Model: " + fileName, e);
|
||||
}
|
||||
|
||||
return customModel;
|
||||
@@ -231,22 +227,15 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
*/
|
||||
protected SearchResponse query(SearchRequest query)
|
||||
{
|
||||
try
|
||||
{
|
||||
return restClient.authenticateUser(testUser).withSearchAPI().search(query);
|
||||
}
|
||||
catch (final Exception exception)
|
||||
{
|
||||
throw new RuntimeException(exception);
|
||||
}
|
||||
return restClient.authenticateUser(testUser).withSearchAPI().search(query);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for Solr to finish indexing and search to return appropriate results
|
||||
*
|
||||
* @param userQuery: Search Query
|
||||
* @param userQuery Search Query
|
||||
* @param contentToFind that's expected to be included / excluded from the results
|
||||
* @param expectedInResults
|
||||
* @param expectedInResults Whether we expect the content in the results or not.
|
||||
* @return true if search returns expected results, i.e. is given content is found or excluded from the results
|
||||
*/
|
||||
public boolean isContentInSearchResults(String userQuery, String contentToFind, boolean expectedInResults) {
|
||||
@@ -254,30 +243,27 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
String expectedStatusCode = HttpStatus.OK.toString();
|
||||
String contentName = (contentToFind == null) ? "" : contentToFind;
|
||||
|
||||
SearchRequest searchRequest = createQuery(userQuery);
|
||||
|
||||
// Repeat search until the query results are as expected or Search Retry count is hit
|
||||
for (int searchCount = 0; searchCount < SEARCH_MAX_ATTEMPS; searchCount++)
|
||||
for (int searchCount = 0; searchCount < SEARCH_MAX_ATTEMPTS; searchCount++)
|
||||
{
|
||||
SearchRequest searchRequest = createQuery(userQuery);
|
||||
SearchResponse response = query(searchRequest);
|
||||
|
||||
if (restClient.getStatusCode().matches(expectedStatusCode))
|
||||
{
|
||||
|
||||
boolean found = response.getEntries().stream()
|
||||
.map(entry -> entry.getModel().getName())
|
||||
.filter(name -> name.equalsIgnoreCase(contentName) || contentName.isBlank())
|
||||
.count() > 0;
|
||||
|
||||
// Loop again if result is not as expected: To cater for solr lag: eventual consistency
|
||||
// Exit loop if result is as expected.
|
||||
if (expectedInResults == found)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Wait for the solr indexing.
|
||||
Utility.waitToLoopTime(properties.getSolrWaitTimeInSeconds(), "Wait For Indexing. Retry Attempt: " + searchCount);
|
||||
}
|
||||
// Wait for the solr indexing (eventual consistency).
|
||||
Utility.waitToLoopTime(properties.getSolrWaitTimeInSeconds(), "Wait For Indexing. Retry Attempt: " + (searchCount + 1));
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -294,9 +280,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
* @param userQuery: search query, this can include the fieldname, unique search string will guarantee accurate results
|
||||
* @param expectedInResults, true if entry is expected in the results set
|
||||
* @return true (indexing is finished) if search returns appropriate results
|
||||
* @throws Exception
|
||||
*/
|
||||
public boolean waitForIndexing(String userQuery, boolean expectedInResults) throws Exception
|
||||
public boolean waitForIndexing(String userQuery, boolean expectedInResults)
|
||||
{
|
||||
// Use the search query as is: fieldname(s) may or may not be specified within the userQuery
|
||||
return waitForIndexing(null, userQuery, expectedInResults);
|
||||
@@ -308,9 +293,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
* @param userQuery
|
||||
* @param expectedInResults
|
||||
* @return
|
||||
* @throws Exception
|
||||
*/
|
||||
public boolean waitForMetadataIndexing(String userQuery, boolean expectedInResults) throws Exception
|
||||
public boolean waitForMetadataIndexing(String userQuery, boolean expectedInResults)
|
||||
{
|
||||
return waitForIndexing("name", userQuery, expectedInResults);
|
||||
}
|
||||
@@ -322,9 +306,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
* @param userQuery
|
||||
* @param expectedInResults
|
||||
* @return
|
||||
* @throws Exception
|
||||
*/
|
||||
public boolean waitForContentIndexing(String userQuery, boolean expectedInResults) throws Exception
|
||||
public boolean waitForContentIndexing(String userQuery, boolean expectedInResults)
|
||||
{
|
||||
return waitForIndexing("cm:content", userQuery, expectedInResults);
|
||||
}
|
||||
@@ -336,9 +319,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
* @param userQuery: search string, unique search string will guarantee accurate results
|
||||
* @param expectedInResults, true if entry is expected in the results set
|
||||
* @return true (indexing is finished) if search returns appropriate results
|
||||
* @throws Exception
|
||||
*/
|
||||
private boolean waitForIndexing(String fieldName, String userQuery, boolean expectedInResults) throws Exception
|
||||
private boolean waitForIndexing(String fieldName, String userQuery, boolean expectedInResults)
|
||||
{
|
||||
String query = (fieldName == null) ? userQuery : String.format("%s:'%s'", fieldName, userQuery);
|
||||
|
||||
@@ -365,16 +347,11 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
*/
|
||||
protected SearchResponse queryAsUser(UserModel user, String queryString)
|
||||
{
|
||||
try {
|
||||
SearchRequest searchRequest = new SearchRequest();
|
||||
RestRequestQueryModel queryModel = new RestRequestQueryModel();
|
||||
queryModel.setQuery(queryString);
|
||||
searchRequest.setQuery(queryModel);
|
||||
return restClient.authenticateUser(user).withSearchAPI().search(searchRequest);
|
||||
} catch (final Exception exception)
|
||||
{
|
||||
throw new RuntimeException(exception);
|
||||
}
|
||||
SearchRequest searchRequest = new SearchRequest();
|
||||
RestRequestQueryModel queryModel = new RestRequestQueryModel();
|
||||
queryModel.setQuery(queryString);
|
||||
searchRequest.setQuery(queryModel);
|
||||
return restClient.authenticateUser(user).withSearchAPI().search(searchRequest);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -384,7 +361,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
* @param queryModel: The queryModel to search for, containing the query
|
||||
* @return the search response from the API
|
||||
*/
|
||||
protected SearchResponse queryAsUser(UserModel user, RestRequestQueryModel queryModel) throws Exception
|
||||
protected SearchResponse queryAsUser(UserModel user, RestRequestQueryModel queryModel)
|
||||
{
|
||||
SearchRequest searchRequest = new SearchRequest();
|
||||
searchRequest.setQuery(queryModel);
|
||||
@@ -396,9 +373,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
|
||||
* Helper method which create an http post request to Search API end point.
|
||||
*
|
||||
* @return {@link SearchResponse} response.
|
||||
* @throws Exception if error
|
||||
*/
|
||||
protected SearchResponse query(RestRequestQueryModel queryReq, RestRequestHighlightModel highlight) throws Exception
|
||||
protected SearchResponse query(RestRequestQueryModel queryReq, RestRequestHighlightModel highlight)
|
||||
{
|
||||
SearchRequest query = new SearchRequest(queryReq);
|
||||
query.setHighlight(highlight);
|
||||
|
||||
+1
-2
@@ -57,9 +57,8 @@ public abstract class AbstractCmisE2ETest extends AbstractE2EFunctionalTest
|
||||
* @param query CMIS Query to be executed
|
||||
* @param expectedCountResults Number of results expected
|
||||
* @return true when results count is equals to expectedCountResults
|
||||
* @throws Exception
|
||||
*/
|
||||
protected boolean waitForIndexing(String query, long expectedCountResults) throws Exception
|
||||
protected boolean waitForIndexing(String query, long expectedCountResults)
|
||||
{
|
||||
|
||||
for (int searchCount = 1; searchCount <= 3; searchCount++)
|
||||
|
||||
-1
@@ -7,7 +7,6 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider;
|
||||
import org.alfresco.utility.model.FileModel;
|
||||
import org.alfresco.utility.model.FolderModel;
|
||||
import org.alfresco.utility.model.QueryModel;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
|
||||
-1
@@ -7,7 +7,6 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider;
|
||||
import org.alfresco.utility.model.FileModel;
|
||||
import org.alfresco.utility.model.FolderModel;
|
||||
import org.alfresco.utility.model.QueryModel;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
|
||||
+2
-3
@@ -5,7 +5,6 @@ import org.alfresco.utility.data.provider.XMLDataConfig;
|
||||
import org.alfresco.utility.data.provider.XMLTestData;
|
||||
import org.alfresco.utility.data.provider.XMLTestDataProvider;
|
||||
import org.alfresco.utility.model.QueryModel;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.annotations.AfterClass;
|
||||
@@ -39,7 +38,7 @@ public class SolrSearchByPathTests extends AbstractCmisE2ETest
|
||||
|
||||
@Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getAllData")
|
||||
@XMLDataConfig(file = "src/test/resources/testdata/search-by-path.xml")
|
||||
public void prepareDataForSearchByPath(XMLTestData testData) throws Exception
|
||||
public void prepareDataForSearchByPath(XMLTestData testData)
|
||||
{
|
||||
this.testData = testData;
|
||||
testData.createUsers(dataUser);
|
||||
@@ -50,7 +49,7 @@ public class SolrSearchByPathTests extends AbstractCmisE2ETest
|
||||
|
||||
@Test(dependsOnMethods = "prepareDataForSearchByPath", dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData")
|
||||
@XMLDataConfig(file = "src/test/resources/testdata/search-by-path.xml")
|
||||
public void executeSearchByPathQueries(QueryModel query) throws Exception
|
||||
public void executeSearchByPathQueries(QueryModel query)
|
||||
{
|
||||
cmisApi.withQuery(query.getValue()).assertResultsCount().equals(query.getResults());
|
||||
}
|
||||
|
||||
+2
-3
@@ -1,13 +1,12 @@
|
||||
package org.alfresco.test.search.functional.searchServices.cmis;
|
||||
|
||||
import org.alfresco.utility.Utility;
|
||||
import org.alfresco.utility.data.provider.XMLDataConfig;
|
||||
import org.alfresco.utility.data.CustomObjectTypeProperties;
|
||||
import org.alfresco.utility.data.provider.XMLDataConfig;
|
||||
import org.alfresco.utility.data.provider.XMLTestDataProvider;
|
||||
import org.alfresco.utility.model.QueryModel;
|
||||
import org.alfresco.utility.model.FileModel;
|
||||
import org.alfresco.utility.model.FolderModel;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.alfresco.utility.model.QueryModel;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
|
||||
-1
@@ -7,7 +7,6 @@ import org.alfresco.utility.model.FileModel;
|
||||
import org.alfresco.utility.model.FileType;
|
||||
import org.alfresco.utility.model.FolderModel;
|
||||
import org.alfresco.utility.model.QueryModel;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.testng.annotations.AfterClass;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
-1
@@ -7,7 +7,6 @@ import org.alfresco.utility.model.FileModel;
|
||||
import org.alfresco.utility.model.FileType;
|
||||
import org.alfresco.utility.model.FolderModel;
|
||||
import org.alfresco.utility.model.QueryModel;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.testng.annotations.AfterClass;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
+1
-1
@@ -43,7 +43,7 @@ public class SolrSearchScoreQueryTests extends AbstractCmisE2ETest
|
||||
* @see org.alfresco.test.search.functional.AbstractE2EFunctionalTest#dataPreparation()
|
||||
*/
|
||||
@Override
|
||||
public void setup() throws Exception
|
||||
public void setup()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
-1
@@ -17,7 +17,6 @@ import org.alfresco.utility.data.DataSite;
|
||||
import org.alfresco.utility.model.FileModel;
|
||||
import org.alfresco.utility.model.FileType;
|
||||
import org.alfresco.utility.model.FolderModel;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.apache.chemistry.opencmis.commons.PropertyIds;
|
||||
import org.apache.chemistry.opencmis.commons.enums.VersioningState;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ public abstract class AbstractSearchServicesE2ETest extends AbstractE2EFunctiona
|
||||
|
||||
protected FileModel file, file2, file3, file4;
|
||||
|
||||
public void searchServicesDataPreparation() throws Exception
|
||||
public void searchServicesDataPreparation()
|
||||
{
|
||||
/*
|
||||
* Create the following file structure for preconditions :
|
||||
|
||||
+2
-4
@@ -18,11 +18,10 @@
|
||||
*/
|
||||
package org.alfresco.test.search.functional.searchServices.search;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.json.Json;
|
||||
import javax.json.JsonObject;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.alfresco.dataprep.SiteService.Visibility;
|
||||
import org.alfresco.rest.search.FacetFieldBucket;
|
||||
@@ -32,7 +31,6 @@ import org.alfresco.rest.search.RestRequestQueryModel;
|
||||
import org.alfresco.rest.search.RestResultBucketsModel;
|
||||
import org.alfresco.rest.search.SearchRequest;
|
||||
import org.alfresco.rest.search.SearchResponse;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
|
||||
import org.alfresco.utility.data.RandomData;
|
||||
import org.alfresco.utility.model.FileModel;
|
||||
|
||||
+4
-5
@@ -18,20 +18,19 @@
|
||||
*/
|
||||
package org.alfresco.test.search.functional.searchServices.search;
|
||||
|
||||
import static org.testng.Assert.assertTrue;
|
||||
|
||||
import org.alfresco.rest.search.SearchNodeModel;
|
||||
import org.alfresco.rest.search.SearchResponse;
|
||||
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
|
||||
import org.alfresco.utility.model.FileModel;
|
||||
import org.alfresco.utility.model.FileType;
|
||||
import org.alfresco.utility.model.FolderModel;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
|
||||
import org.alfresco.utility.report.Bug;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static org.testng.Assert.assertTrue;
|
||||
|
||||
/**
|
||||
* Search end point Public API test with finger print.
|
||||
*
|
||||
@@ -80,7 +79,7 @@ public class FingerPrintTest extends AbstractE2EFunctionalTest
|
||||
public void makeSureFingerprintQueryWorksAfterMetadataUpdate() throws Exception
|
||||
{
|
||||
// Index a new file with content
|
||||
FileModel file = new FileModel("Project_Contract.pdf", FileType.TEXT_PLAIN, "A content which is completely different from other indexed files.");
|
||||
FileModel file = new FileModel("Project_Contract.txt", FileType.TEXT_PLAIN, "A content which is completely different from other indexed files.");
|
||||
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file);
|
||||
|
||||
// make sure the content has been indexed (i.e. the ContentTracker fingerprint has been correctly computed
|
||||
|
||||
-1
@@ -22,7 +22,6 @@ import org.alfresco.rest.search.RestRequestFieldsModel;
|
||||
import org.alfresco.rest.search.RestRequestHighlightModel;
|
||||
import org.alfresco.rest.search.RestRequestQueryModel;
|
||||
import org.alfresco.rest.search.SearchResponse;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.alfresco.utility.report.Bug;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
+1
-1
@@ -72,7 +72,7 @@ public class SearchTest extends AbstractSearchServicesE2ETest
|
||||
|
||||
SearchNodeModel entity = nodes.getEntryByIndex(0);
|
||||
entity.assertThat().field("search").contains("score");
|
||||
entity.getSearch().assertThat().field("score").isNotEmpty();
|
||||
entity.getSearch().assertThat().field("score").isNotNull();
|
||||
Assert.assertEquals(entity.getName(),"pangram.txt");
|
||||
}
|
||||
|
||||
|
||||
-1
@@ -18,7 +18,6 @@ import org.alfresco.utility.data.DataSite;
|
||||
import org.alfresco.utility.model.FileModel;
|
||||
import org.alfresco.utility.model.FileType;
|
||||
import org.alfresco.utility.model.FolderModel;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.apache.chemistry.opencmis.commons.PropertyIds;
|
||||
import org.apache.chemistry.opencmis.commons.enums.VersioningState;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
+1
-3
@@ -14,14 +14,12 @@
|
||||
*/
|
||||
package org.alfresco.test.search.functional.searchServices.solr;
|
||||
|
||||
import java.net.URLEncoder;
|
||||
|
||||
import javax.json.JsonArrayBuilder;
|
||||
import java.net.URLEncoder;
|
||||
|
||||
import org.alfresco.rest.core.JsonBodyGenerator;
|
||||
import org.alfresco.rest.model.RestTextResponse;
|
||||
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
|
||||
import org.alfresco.search.TestGroup;
|
||||
import org.hamcrest.Matchers;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.testng.Assert;
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ public abstract class AbstractUpgradeE2ETest extends AbstractTestNGSpringContext
|
||||
protected XMLTestData testData;
|
||||
|
||||
@BeforeClass(alwaysRun = true)
|
||||
public void checkServerHealth() throws Exception
|
||||
public void checkServerHealth()
|
||||
{
|
||||
serverHealth.assertServerIsOnline();
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<version>10</version>
|
||||
</parent>
|
||||
<artifactId>alfresco-search-and-insight-parent</artifactId>
|
||||
<version>1.4.0-SNAPSHOT</version>
|
||||
<version>1.1.0</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>Alfresco Search And Insight Parent</name>
|
||||
<distributionManagement>
|
||||
@@ -24,7 +24,7 @@
|
||||
<connection>scm:git:https://git.alfresco.com/search_discovery/insightengine.git</connection>
|
||||
<developerConnection>scm:git:https://git.alfresco.com/search_discovery/insightengine.git</developerConnection>
|
||||
<url>https://git.alfresco.com/search_discovery/insightengine.git</url>
|
||||
<tag>HEAD</tag>
|
||||
<tag>1.1.0</tag>
|
||||
</scm>
|
||||
<properties>
|
||||
<solr.base.version>6.6.5</solr.base.version>
|
||||
|
||||
@@ -100,7 +100,7 @@ SOLR_SSL_WANT_CLIENT_AUTH=false
|
||||
Once this settings are ready, start SOLR service from command line:
|
||||
|
||||
```
|
||||
$ ./solr/bin/solr start "-Dcreate.alfresco.defaults=alfresco,archive \
|
||||
$ ./solr/bin/solr start -a "-Dcreate.alfresco.defaults=alfresco,archive \
|
||||
-Dsolr.ssl.checkPeerName=false \
|
||||
-Dsolr.allow.unsafe.resourceloading=true" -f
|
||||
```
|
||||
@@ -130,7 +130,7 @@ DBID based sharding can be set up from the command line. For example a core cont
|
||||
index with twelve shards can be set up by starting an instance of Alfresco Search Services with a command like:
|
||||
|
||||
```bash
|
||||
./bin/solr start -a -Dcreate.alfresco.defaults=alfresco,archive -Dnum.shards=12 -Dshard.ids=0,1,6,7
|
||||
./bin/solr start -a "-Dcreate.alfresco.defaults=alfresco,archive -Dnum.shards=12 -Dshard.ids=0,1,6,7"
|
||||
```
|
||||
|
||||
Further instances should be set up to contain the other shards, and it is possible to adjust the distribution and
|
||||
@@ -169,7 +169,7 @@ $ sed -i 's/alfresco.secureComms=https/alfresco.secureComms=none/' solrhome/temp
|
||||
Start SOLR service from command line:
|
||||
|
||||
```
|
||||
$ ./solr/bin/solr start "-Dcreate.alfresco.defaults=alfresco,archive" -f
|
||||
$ ./solr/bin/solr start -a "-Dcreate.alfresco.defaults=alfresco,archive" -f
|
||||
```
|
||||
|
||||
SOLR will create Alfresco cores (`alfresco` and `archive`) when starting, and configuration from `rerank` template will be copied to each core and if you also started an ACS instance running in [http://localhost:8080/alfresco](http://localhost:8080/alfresco) then the index will be populated.
|
||||
@@ -247,14 +247,7 @@ $ docker run -p 8983:8983 \
|
||||
-e SOLR_SSL_TRUST_STORE_TYPE=JCEKS \
|
||||
-e SOLR_SSL_NEED_CLIENT_AUTH=true \
|
||||
-e SOLR_OPTS="-Dsolr.ssl.checkPeerName=false \
|
||||
-Dsolr.allow.unsafe.resourceloading=true \
|
||||
-Dalfresco.encryption.ssl.keystore.type=JCEKS
|
||||
-Dalfresco.encryption.ssl.keystore.location=/opt/alfresco-search-services/keystores/ssl.repo.client.keystore
|
||||
-Dalfresco.encryption.ssl.keystore.passwordFileLocation=/opt/alfresco-search-services/keystores/ssl-keystore-passwords.properties
|
||||
-Dalfresco.encryption.ssl.truststore.type=JCEKS
|
||||
-Dalfresco.encryption.ssl.truststore.location=/opt/alfresco-search-services/keystores/ssl.repo.client.truststore
|
||||
-Dalfresco.encryption.ssl.truststore.passwordFileLocation=/opt/alfresco-search-services/keystores/ssl-truststore-passwords.properties
|
||||
" \
|
||||
-Dsolr.allow.unsafe.resourceloading=true" \
|
||||
searchservices:develop
|
||||
```
|
||||
|
||||
@@ -345,12 +338,6 @@ solr6:
|
||||
SOLR_OPTS: "
|
||||
-Dsolr.ssl.checkPeerName=false
|
||||
-Dsolr.allow.unsafe.resourceloading=true
|
||||
-Dalfresco.encryption.ssl.keystore.type=JCEKS
|
||||
-Dalfresco.encryption.ssl.keystore.location=/opt/alfresco-search-services/keystores/ssl.repo.client.keystore
|
||||
-Dalfresco.encryption.ssl.keystore.passwordFileLocation=/opt/alfresco-search-services/keystores/ssl-keystore-passwords.properties
|
||||
-Dalfresco.encryption.ssl.truststore.type=JCEKS
|
||||
-Dalfresco.encryption.ssl.truststore.location=/opt/alfresco-search-services/keystores/ssl.repo.client.truststore
|
||||
-Dalfresco.encryption.ssl.truststore.passwordFileLocation=/opt/alfresco-search-services/keystores/ssl-truststore-passwords.properties
|
||||
"
|
||||
ports:
|
||||
- 8083:8983 #Browser port
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
@startuml
|
||||
|
||||
database "Solr" {
|
||||
package "SolrInfrmationServer" {
|
||||
[UpdateIndex]
|
||||
}
|
||||
package "org.alfresco.tracker.content"{
|
||||
[ContentConsumer]
|
||||
[ContentMessageProducer]
|
||||
[SharedFileStoreClient]
|
||||
}
|
||||
folder "ContentStore" {
|
||||
[UpdateFile]
|
||||
}
|
||||
}
|
||||
node "Alfresco SharedFileStore" {
|
||||
[GetFile]
|
||||
}
|
||||
|
||||
database "ActiveMQ" {
|
||||
[ContentMessageQue]
|
||||
}
|
||||
|
||||
[ContentMessageProducer] --> [ContentMessageQue]
|
||||
[ContentConsumer] --> [ContentMessageQue]
|
||||
[ContentConsumer] --> [SharedFileStoreClient]
|
||||
[ContentConsumer] <-- [SharedFileStoreClient]
|
||||
[SharedFileStoreClient] --> [GetFile]
|
||||
[ContentConsumer] --> [UpdateIndex]
|
||||
[UpdateIndex] -->[UpdateFile]
|
||||
|
||||
|
||||
@enduml
|
||||
@@ -149,6 +149,19 @@ where
|
||||
|
||||
|
||||
##### Last Registered Indexing Shard (LRIS)
|
||||
|
||||
> WARNING: This is an experimental feature that is subject to change.
|
||||
The current known issues are:
|
||||
>
|
||||
> \- Incompatibility with the "Purge" action on the Alfresco Admin Console
|
||||
> \- Incompatibility with the "Purge on Startup" option on the Alfresco Admin Console
|
||||
>
|
||||
> As a result, when this router is used and the purge action is invoked you may experience a wrong shard subscription
|
||||
> registration order. That would cause a wrong distribution of the indexed data across the cluster.
|
||||
>
|
||||
> Note that if "Purge on restart" option on the Admin Console is checked by default, then before building a
|
||||
> Solr cluster which uses this router, please make sure the check has been disabled.
|
||||
|
||||
This method uses, as the name suggests, the last indexing shard which subscribed to the Shard Registry.
|
||||
Like the _DB_ID_RANGE_ strategy, it's possible to add new shards to the cluster without requiring a reindex. And as consequence of that, the total number of shards (i.e. the _shard.count_ attribute) isn't needed with this sharding strategy.
|
||||
At indexing time, when the _MetadataTracker_ periodically asks for transactions and nodes, Alfresco repository creates the nodes instances and, using the associated transaction timestamp, asks to the _ShardRegistry_ which is the target Shard which should index/own the node.
|
||||
@@ -212,6 +225,19 @@ where
|
||||
* shard.count is the total number of the shards composing the cluster (required by the DBID fallback strategy)
|
||||
|
||||
##### Explicit Shard ID with fallback on LRIS (EXPLICIT_ID_FALLBACK_LRIS)
|
||||
|
||||
> WARNING: This is an experimental feature that is subject to change.
|
||||
The current known issues are:
|
||||
>
|
||||
> \- Incompatibility with the "Purge" action on the Alfresco Admin Console
|
||||
> \- Incompatibility with the "Purge on Startup" option on the Alfresco Admin Console
|
||||
>
|
||||
> As a result, when this router is used and the purge action is invoked you may experience a wrong shard subscription
|
||||
> registration order. That would cause a wrong distribution of the indexed data across the cluster.
|
||||
>
|
||||
> Note that if "Purge on restart" option on the Admin Console is checked by default, then before building a
|
||||
> Solr cluster which uses this router, please make sure the check has been disabled.
|
||||
|
||||
This method still belongs to the "explicit" id family, where nodes are routed to shards accordingly to a value of a node property which is supposed to contain the target shard instance identifier.
|
||||
So it is very similar to the previous one. The important difference resides in the fallback strategy: in case the target shard cannot be determined used the supplied node property, the
|
||||
"Last Registered Indexing Shard" is used as fallback.
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
<parent>
|
||||
<groupId>org.alfresco</groupId>
|
||||
<artifactId>alfresco-search-parent</artifactId>
|
||||
<version>1.4.0-SNAPSHOT</version>
|
||||
<version>1.1.0</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
<dependency>
|
||||
<groupId>org.alfresco</groupId>
|
||||
<artifactId>alfresco-solrclient-lib</artifactId>
|
||||
<version>1.4.0-SNAPSHOT</version>
|
||||
<version>1.1.0</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>servlet-api</artifactId>
|
||||
@@ -75,13 +75,13 @@
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.26</version>
|
||||
<version>1.7.27</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>1.7.26</version>
|
||||
<version>1.7.27</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
+1
-1
@@ -951,7 +951,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
|
||||
{
|
||||
if (maxNodeId >= midpoint)
|
||||
{
|
||||
if(density >= 1)
|
||||
if(density >= 1 || density == 0)
|
||||
{
|
||||
//This is fully dense shard. I'm not sure if it's possible to have more nodes on the shards
|
||||
//then the offset, but if it does happen don't expand.
|
||||
|
||||
+1
-2
@@ -1195,8 +1195,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
|
||||
|
||||
public void removeModel(QName modelQName)
|
||||
{
|
||||
// FIXME: this has no effect. The method should be changed (SEARCH-1482)
|
||||
modelErrors.remove(modelQName);
|
||||
modelErrors.remove(getM2Model(modelQName).getName());
|
||||
dictionaryDAO.removeModel(modelQName);
|
||||
}
|
||||
|
||||
|
||||
+13
-6
@@ -957,9 +957,10 @@ public class SolrInformationServer implements InformationServer
|
||||
SolrIndexSearcher solrIndexSearcher = refCounted.get();
|
||||
coreSummary.add("Searcher", solrIndexSearcher.getStatistics());
|
||||
Map<String, SolrInfoMBean> infoRegistry = core.getInfoRegistry();
|
||||
for (String key : infoRegistry.keySet())
|
||||
for (Entry<String, SolrInfoMBean> infos : infoRegistry.entrySet())
|
||||
{
|
||||
SolrInfoMBean infoMBean = infoRegistry.get(key);
|
||||
SolrInfoMBean infoMBean = infos.getValue();
|
||||
String key = infos.getKey();
|
||||
if (key.equals("/alfresco"))
|
||||
{
|
||||
// TODO Do we really need to fixStats in solr4?
|
||||
@@ -2117,8 +2118,9 @@ public class SolrInformationServer implements InformationServer
|
||||
static void addPropertiesToDoc(Map<QName, PropertyValue> properties, boolean isContentIndexedForNode,
|
||||
SolrInputDocument newDoc, SolrInputDocument cachedDoc, boolean transformContentFlag)
|
||||
{
|
||||
for (QName propertyQName : properties.keySet())
|
||||
for (Entry<QName, PropertyValue> property : properties.entrySet())
|
||||
{
|
||||
QName propertyQName = property.getKey();
|
||||
newDoc.addField(FIELD_PROPERTIES, propertyQName.toString());
|
||||
newDoc.addField(FIELD_PROPERTIES, propertyQName.getPrefixString());
|
||||
|
||||
@@ -3412,10 +3414,15 @@ public class SolrInformationServer implements InformationServer
|
||||
SolrQueryRequest request, UpdateRequestProcessor processor, LinkedHashSet<Long> stack)
|
||||
throws AuthenticationException, IOException, JSONException
|
||||
{
|
||||
if ((skipDescendantDocsForSpecificTypes && typesForSkippingDescendantDocs.contains(parentNodeMetaData.getType())) ||
|
||||
(skipDescendantDocsForSpecificAspects && shouldBeIgnoredByAnyAspect(parentNodeMetaData.getAspects())))
|
||||
|
||||
// skipDescendantDocsForSpecificAspects is initialised on a synchronised method, so access must be also synchronised
|
||||
synchronized (this)
|
||||
{
|
||||
return;
|
||||
if ((skipDescendantDocsForSpecificTypes && typesForSkippingDescendantDocs.contains(parentNodeMetaData.getType())) ||
|
||||
(skipDescendantDocsForSpecificAspects && shouldBeIgnoredByAnyAspect(parentNodeMetaData.getAspects())))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Set<Long> childIds = new HashSet<>();
|
||||
|
||||
+4
-2
@@ -41,6 +41,7 @@ import java.util.Iterator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
@@ -472,8 +473,9 @@ public class AsyncBuildSuggestComponent extends SearchComponent implements SolrC
|
||||
@Override
|
||||
public long ramBytesUsed() {
|
||||
long sizeInBytes = 0;
|
||||
for (String key : suggesters.keySet()) {
|
||||
sizeInBytes += suggesters.get(key).get(ASYNC_CACHE_KEY).ramBytesUsed();
|
||||
for (Entry<String, SuggesterCache> suggester : suggesters.entrySet())
|
||||
{
|
||||
sizeInBytes += suggester.getValue().get(ASYNC_CACHE_KEY).ramBytesUsed();
|
||||
}
|
||||
return sizeInBytes;
|
||||
}
|
||||
|
||||
+1
-1
@@ -91,7 +91,7 @@ public class RewriteFacetParametersComponent extends SearchComponent
|
||||
String rows = params.get("rows");
|
||||
if(rows != null && !rows.isEmpty())
|
||||
{
|
||||
Integer row = new Integer(rows);
|
||||
Integer row = Integer.valueOf(rows);
|
||||
// Avoid +1 in SOLR code which produces null:java.lang.NegativeArraySizeException at at org.apache.lucene.util.PriorityQueue.<init>(PriorityQueue.java:56)
|
||||
if(row > 1000000)
|
||||
{
|
||||
|
||||
+4
-5
@@ -19,7 +19,6 @@
|
||||
package org.alfresco.solr.component;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.DirectoryStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
@@ -53,9 +52,9 @@ public class TempFileWarningLogger
|
||||
log.debug("Looking for temp files matching " + glob + " in directory " + dir);
|
||||
}
|
||||
|
||||
try(DirectoryStream<Path> stream = Files.newDirectoryStream(dir, glob))
|
||||
try
|
||||
{
|
||||
for (Path file : stream)
|
||||
for (Path file : Files.newDirectoryStream(dir, glob))
|
||||
{
|
||||
if (log.isDebugEnabled())
|
||||
{
|
||||
@@ -74,9 +73,9 @@ public class TempFileWarningLogger
|
||||
|
||||
public void removeFiles()
|
||||
{
|
||||
try(DirectoryStream<Path> stream = Files.newDirectoryStream(dir, glob))
|
||||
try
|
||||
{
|
||||
for (Path file : stream)
|
||||
for (Path file : Files.newDirectoryStream(dir, glob))
|
||||
{
|
||||
file.toFile().delete();
|
||||
}
|
||||
|
||||
+64
-25
@@ -64,6 +64,8 @@ public class SolrCoreLoadRegistration {
|
||||
public static void registerForCore(AlfrescoCoreAdminHandler adminHandler, CoreContainer coreContainer, SolrCore core,
|
||||
String coreName)
|
||||
{
|
||||
|
||||
|
||||
TrackerRegistry trackerRegistry = adminHandler.getTrackerRegistry();
|
||||
Properties props = new CoreDescriptorDecorator(core.getCoreDescriptor()).getProperties();
|
||||
//Prepare cores
|
||||
@@ -79,10 +81,23 @@ public class SolrCoreLoadRegistration {
|
||||
props.putAll(srv.getProps());
|
||||
adminHandler.getInformationServers().put(coreName, srv);
|
||||
|
||||
SolrTrackerScheduler scheduler = adminHandler.getScheduler();
|
||||
|
||||
// Prevents other threads from registering the ModelTracker at the same time
|
||||
// Create model tracker and load all the persisted models
|
||||
createModelTracker(coreName,
|
||||
trackerRegistry,
|
||||
props,
|
||||
coreContainer.getSolrHome(),
|
||||
repositoryClient,
|
||||
srv,
|
||||
scheduler);
|
||||
|
||||
|
||||
log.info("Starting to track " + coreName);
|
||||
if (Boolean.parseBoolean(props.getProperty("enable.alfresco.tracking", "false")))
|
||||
{
|
||||
SolrTrackerScheduler scheduler = adminHandler.getScheduler();
|
||||
|
||||
if (trackerRegistry.hasTrackersForCore(coreName))
|
||||
{
|
||||
log.info("Trackers for " + coreName+ " is already registered, shutting them down.");
|
||||
@@ -91,28 +106,7 @@ public class SolrCoreLoadRegistration {
|
||||
adminHandler.getInformationServers().remove(coreName);
|
||||
}
|
||||
|
||||
// Prevents other threads from registering the ModelTracker at the same time
|
||||
synchronized (SolrCoreLoadRegistration.class)
|
||||
{
|
||||
ModelTracker mTracker = trackerRegistry.getModelTracker();
|
||||
if (mTracker == null)
|
||||
{
|
||||
log.debug("Creating ModelTracker when registering trackers for core " + coreName);
|
||||
mTracker = new ModelTracker(coreContainer.getSolrHome(), props, repositoryClient,
|
||||
coreName, srv);
|
||||
|
||||
trackerRegistry.setModelTracker(mTracker);
|
||||
|
||||
log.info("Ensuring first model sync.");
|
||||
mTracker.ensureFirstModelSync();
|
||||
log.info("Done ensuring first model sync.");
|
||||
|
||||
//Scheduling the ModelTracker.
|
||||
scheduler.schedule(mTracker, coreName, props);
|
||||
}
|
||||
}
|
||||
|
||||
List<Tracker> trackers = createTrackers(coreName, trackerRegistry, props, scheduler, repositoryClient, srv);
|
||||
List<Tracker> trackers = createCoreTrackers(coreName, trackerRegistry, props, scheduler, repositoryClient, srv);
|
||||
|
||||
CommitTracker commitTracker = new CommitTracker(props, repositoryClient, coreName, srv, trackers);
|
||||
trackerRegistry.register(coreName, commitTracker);
|
||||
@@ -120,7 +114,8 @@ public class SolrCoreLoadRegistration {
|
||||
log.info("The Trackers are now scheduled to run");
|
||||
trackers.add(commitTracker); //Add the commitTracker to the list of scheduled trackers that can be shutdown
|
||||
|
||||
core.addCloseHook(new CloseHook() {
|
||||
core.addCloseHook(new CloseHook()
|
||||
{
|
||||
@Override
|
||||
public void preClose(SolrCore core)
|
||||
{
|
||||
@@ -148,7 +143,12 @@ public class SolrCoreLoadRegistration {
|
||||
* @param srv
|
||||
* @return A list of trackers
|
||||
*/
|
||||
private static List<Tracker> createTrackers(String coreName, TrackerRegistry trackerRegistry, Properties props, SolrTrackerScheduler scheduler, SOLRAPIClient repositoryClient, SolrInformationServer srv) {
|
||||
private static List<Tracker> createCoreTrackers(String coreName,
|
||||
TrackerRegistry trackerRegistry,
|
||||
Properties props,
|
||||
SolrTrackerScheduler scheduler,
|
||||
SOLRAPIClient repositoryClient,
|
||||
SolrInformationServer srv) {
|
||||
List<Tracker> trackers = new ArrayList<Tracker>();
|
||||
|
||||
AclTracker aclTracker = new AclTracker(props, repositoryClient, coreName, srv);
|
||||
@@ -178,6 +178,45 @@ public class SolrCoreLoadRegistration {
|
||||
return trackers;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create model tracker and load persisted models.
|
||||
*
|
||||
* @param coreName
|
||||
* @param trackerRegistry
|
||||
* @param props
|
||||
* @param solrHome
|
||||
* @param repositoryClient
|
||||
* @param srv
|
||||
* @param scheduler
|
||||
* @return true if model tracker has been created, false if it already exists.
|
||||
*/
|
||||
private synchronized static void createModelTracker(String coreName,
|
||||
TrackerRegistry trackerRegistry,
|
||||
Properties props,
|
||||
String solrHome,
|
||||
SOLRAPIClient repositoryClient,
|
||||
SolrInformationServer srv,
|
||||
SolrTrackerScheduler scheduler)
|
||||
{
|
||||
ModelTracker mTracker = trackerRegistry.getModelTracker();
|
||||
if (mTracker == null)
|
||||
{
|
||||
log.debug("Creating ModelTracker");
|
||||
mTracker = new ModelTracker(solrHome, props, repositoryClient,
|
||||
coreName, srv);
|
||||
|
||||
trackerRegistry.setModelTracker(mTracker);
|
||||
log.info("Ensuring first model sync.");
|
||||
mTracker.ensureFirstModelSync();
|
||||
log.info("Done ensuring first model sync.");
|
||||
|
||||
//Scheduling the ModelTracker.
|
||||
scheduler.schedule(mTracker, coreName, props);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Shuts down the trackers for a core.
|
||||
*
|
||||
|
||||
+1
-1
@@ -47,7 +47,7 @@ public abstract class AbstractSolrCachingScorer extends Scorer
|
||||
|
||||
static {
|
||||
for(int i = 0; i < cache.length; i++)
|
||||
cache[i] = new Long(i);
|
||||
cache[i] = Long.valueOf(i);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+3
-3
@@ -20,6 +20,7 @@ package org.alfresco.solr.query;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.alfresco.solr.AlfrescoSolrDataModel;
|
||||
import org.alfresco.solr.AlfrescoSolrDataModel.FieldUse;
|
||||
@@ -111,10 +112,9 @@ public class MimetypeGroupingCollector extends DelegatingCollector
|
||||
rb.rsp.add("analytics", analytics);
|
||||
NamedList<Object> fieldCounts = new NamedList<>();
|
||||
analytics.add("mimetype()", fieldCounts);
|
||||
for(String key : counters.keySet())
|
||||
for (Entry<String, Counter> counter : counters.entrySet())
|
||||
{
|
||||
Counter counter = counters.get(key);
|
||||
fieldCounts.add(key, counter.get());
|
||||
fieldCounts.add(counter.getKey(), counter.getValue().get());
|
||||
}
|
||||
|
||||
if(this.delegate instanceof DelegatingCollector) {
|
||||
|
||||
+29
-36
@@ -3301,44 +3301,35 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
|
||||
|
||||
protected String getToken(String field, String value, AnalysisMode analysisMode) throws ParseException
|
||||
{
|
||||
try (TokenStream source = getAnalyzer().tokenStream(field, new StringReader(value)))
|
||||
{
|
||||
String tokenised = null;
|
||||
|
||||
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(value));
|
||||
|
||||
while (source.incrementToken())
|
||||
{
|
||||
CharTermAttribute cta = source.getAttribute(CharTermAttribute.class);
|
||||
OffsetAttribute offsetAtt = source.getAttribute(OffsetAttribute.class);
|
||||
TypeAttribute typeAtt = null;
|
||||
if (source.hasAttribute(TypeAttribute.class))
|
||||
{
|
||||
typeAtt = source.getAttribute(TypeAttribute.class);
|
||||
}
|
||||
PositionIncrementAttribute posIncAtt = null;
|
||||
if (source.hasAttribute(PositionIncrementAttribute.class))
|
||||
{
|
||||
posIncAtt = source.getAttribute(PositionIncrementAttribute.class);
|
||||
}
|
||||
PackedTokenAttributeImpl token = new PackedTokenAttributeImpl();
|
||||
token.setEmpty().copyBuffer(cta.buffer(), 0, cta.length());
|
||||
token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
|
||||
if (typeAtt != null)
|
||||
{
|
||||
token.setType(typeAtt.type());
|
||||
}
|
||||
if (posIncAtt != null)
|
||||
{
|
||||
token.setPositionIncrement(posIncAtt.getPositionIncrement());
|
||||
}
|
||||
|
||||
tokenised = token.toString();
|
||||
}
|
||||
return tokenised;
|
||||
} catch (IOException e)
|
||||
CharTermAttribute cta = source.getAttribute(CharTermAttribute.class);
|
||||
OffsetAttribute offsetAtt = source.getAttribute(OffsetAttribute.class);
|
||||
TypeAttribute typeAtt = null;
|
||||
if (source.hasAttribute(TypeAttribute.class))
|
||||
{
|
||||
throw new ParseException("IO" + e.getMessage());
|
||||
typeAtt = source.getAttribute(TypeAttribute.class);
|
||||
}
|
||||
PositionIncrementAttribute posIncAtt = null;
|
||||
if (source.hasAttribute(PositionIncrementAttribute.class))
|
||||
{
|
||||
posIncAtt = source.getAttribute(PositionIncrementAttribute.class);
|
||||
}
|
||||
PackedTokenAttributeImpl token = new PackedTokenAttributeImpl();
|
||||
token.setEmpty().copyBuffer(cta.buffer(), 0, cta.length());
|
||||
token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
|
||||
if (typeAtt != null)
|
||||
{
|
||||
token.setType(typeAtt.type());
|
||||
}
|
||||
if (posIncAtt != null)
|
||||
{
|
||||
token.setPositionIncrement(posIncAtt.getPositionIncrement());
|
||||
}
|
||||
|
||||
return token.toString();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -5472,11 +5463,13 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
|
||||
}
|
||||
|
||||
protected BytesRef analyzeMultitermTerm(String field, String part, Analyzer analyzerIn) {
|
||||
|
||||
if (analyzerIn == null) analyzerIn = getAnalyzer();
|
||||
|
||||
try (TokenStream source = analyzerIn.tokenStream(field, part)) {
|
||||
source.reset();
|
||||
try
|
||||
{
|
||||
|
||||
TokenStream source = analyzerIn.tokenStream(field, part);
|
||||
TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
|
||||
|
||||
if (!source.incrementToken())
|
||||
|
||||
+8
-1
@@ -55,6 +55,13 @@ public class DateQuarterRouter implements DocRouter
|
||||
calendar.setTime(date);
|
||||
int month = calendar.get(Calendar.MONTH);
|
||||
int year = calendar.get(Calendar.YEAR);
|
||||
return Math.ceil(((year * 12) + (month+1)) / 3) % numShards == shardInstance;
|
||||
|
||||
// Avoid using Math.ceil with Integer
|
||||
int countMonths = ((year * 12) + (month+1));
|
||||
int grouping = 3;
|
||||
int ceilGroupInstance = countMonths / grouping + ((countMonths % grouping == 0) ? 0 : 1);
|
||||
|
||||
return ceilGroupInstance % numShards == shardInstance;
|
||||
|
||||
}
|
||||
}
|
||||
+1
-6
@@ -37,17 +37,12 @@ import org.alfresco.solr.client.Node;
|
||||
* As a result, when this router is used and the purge action is invoked you may experience a wrong shard subscription
|
||||
* registration order. That would cause a wrong distribution of the indexed data across the cluster.
|
||||
*
|
||||
* Note the "Purge on restart" option on the Admin Console is checked by default, so before building a
|
||||
* Note that if "Purge on restart" option on the Admin Console is checked by default, then before building a
|
||||
* Solr cluster which uses this router, please make sure the check has been disabled.
|
||||
*
|
||||
* </b><br/><br/>
|
||||
* </p>
|
||||
*
|
||||
* Specifically, until the whole feature will be officially released, the LRIS document routing feature is not compatible
|
||||
* with the "Purge" action on the Alfresco Admin Console.
|
||||
* Note that at time of writing, the "Purge on startup" option in the Admin Console is enabled by default so prior to
|
||||
* build your search cluster, you have to make sure that option is unchecked.
|
||||
*
|
||||
* @author Elia
|
||||
* @author agazzarini
|
||||
* @since 1.4
|
||||
|
||||
+20
-6
@@ -81,6 +81,9 @@ public class MetadataTracker extends AbstractTracker implements Tracker
|
||||
private ConcurrentLinkedQueue<Long> nodesToPurge = new ConcurrentLinkedQueue<Long>();
|
||||
private ConcurrentLinkedQueue<String> queriesToReindex = new ConcurrentLinkedQueue<String>();
|
||||
private DocRouter docRouter;
|
||||
/** The string representation of the shard key. */
|
||||
private String shardKey;
|
||||
/** The property to use for determining the shard. */
|
||||
private QName shardProperty;
|
||||
|
||||
public MetadataTracker(Properties p, SOLRAPIClient client, String coreName,
|
||||
@@ -89,16 +92,24 @@ public class MetadataTracker extends AbstractTracker implements Tracker
|
||||
super(p, client, coreName, informationServer, Tracker.Type.MetaData);
|
||||
transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100"));
|
||||
shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID);
|
||||
String shardKey = p.getProperty(SHARD_KEY_KEY);
|
||||
if(shardKey != null)
|
||||
{
|
||||
shardProperty = getShardProperty(shardKey);
|
||||
}
|
||||
shardKey = p.getProperty(SHARD_KEY_KEY);
|
||||
updateShardProperty();
|
||||
docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod));
|
||||
nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10"));
|
||||
threadHandler = new ThreadHandler(p, coreName, "MetadataTracker");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Set the shard property using the shard key.
|
||||
*/
|
||||
private void updateShardProperty()
|
||||
{
|
||||
if(shardProperty == null && shardKey != null)
|
||||
{
|
||||
shardProperty = getShardProperty(shardKey);
|
||||
}
|
||||
}
|
||||
|
||||
MetadataTracker()
|
||||
{
|
||||
super(Tracker.Type.MetaData);
|
||||
@@ -228,6 +239,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker
|
||||
HashMap<String, String> propertyBag = new HashMap<>();
|
||||
propertyBag.put("coreName", coreName);
|
||||
HashMap<String, String> extendedPropertyBag = new HashMap<>(propertyBag);
|
||||
updateShardProperty();
|
||||
extendedPropertyBag.putAll(docRouter.getProperties(shardProperty));
|
||||
|
||||
return ShardStateBuilder.shardState()
|
||||
@@ -369,6 +381,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker
|
||||
gnp.setTransactionIds(txs);
|
||||
gnp.setStoreProtocol(storeRef.getProtocol());
|
||||
gnp.setStoreIdentifier(storeRef.getIdentifier());
|
||||
updateShardProperty();
|
||||
gnp.setShardProperty(shardProperty);
|
||||
gnp.setCoreName(coreName);
|
||||
|
||||
@@ -889,6 +902,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker
|
||||
gnp.setTransactionIds(txIds);
|
||||
gnp.setStoreProtocol(storeRef.getProtocol());
|
||||
gnp.setStoreIdentifier(storeRef.getIdentifier());
|
||||
updateShardProperty();
|
||||
gnp.setShardProperty(shardProperty);
|
||||
gnp.setCoreName(coreName);
|
||||
List<Node> nodes = client.getNodes(gnp, Integer.MAX_VALUE);
|
||||
|
||||
+39
-64
@@ -83,18 +83,18 @@ import org.json.JSONException;
|
||||
public class ModelTracker extends AbstractTracker implements Tracker
|
||||
{
|
||||
|
||||
private Set<StoreRef> indexedStores = new HashSet<StoreRef>();
|
||||
private Set<StoreRef> ignoredStores = new HashSet<StoreRef>();
|
||||
private Set<String> indexedTenants = new HashSet<String>();
|
||||
private Set<String> ignoredTenants = new HashSet<String>();
|
||||
private Set<QName> indexedDataTypes = new HashSet<QName>();
|
||||
private Set<QName> ignoredDataTypes = new HashSet<QName>();
|
||||
private Set<QName> indexedTypes = new HashSet<QName>();
|
||||
private Set<QName> ignoredTypes = new HashSet<QName>();
|
||||
private Set<QName> indexedAspects = new HashSet<QName>();
|
||||
private Set<QName> ignoredAspects = new HashSet<QName>();
|
||||
private Set<String> indexedFields = new HashSet<String>();
|
||||
private Set<String> ignoredFields = new HashSet<String>();
|
||||
private final Set<StoreRef> indexedStores = new HashSet<>();
|
||||
private final Set<StoreRef> ignoredStores = new HashSet<>();
|
||||
private final Set<String> indexedTenants = new HashSet<>();
|
||||
private final Set<String> ignoredTenants = new HashSet<>();
|
||||
private final Set<QName> indexedDataTypes = new HashSet<>();
|
||||
private final Set<QName> ignoredDataTypes = new HashSet<>();
|
||||
private final Set<QName> indexedTypes = new HashSet<>();
|
||||
private final Set<QName> ignoredTypes = new HashSet<>();
|
||||
private final Set<QName> indexedAspects = new HashSet<>();
|
||||
private final Set<QName> ignoredAspects = new HashSet<>();
|
||||
private final Set<String> indexedFields = new HashSet<>();
|
||||
private final Set<String> ignoredFields = new HashSet<>();
|
||||
|
||||
private ReentrantReadWriteLock modelLock = new ReentrantReadWriteLock();
|
||||
private volatile boolean hasModels = false;
|
||||
@@ -111,7 +111,7 @@ public class ModelTracker extends AbstractTracker implements Tracker
|
||||
{
|
||||
alfrescoModelDir.mkdir();
|
||||
}
|
||||
|
||||
|
||||
loadPersistedModels();
|
||||
}
|
||||
|
||||
@@ -124,23 +124,15 @@ public class ModelTracker extends AbstractTracker implements Tracker
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
*/
|
||||
private void loadPersistedModels()
|
||||
{
|
||||
HashMap<String, M2Model> modelMap = new HashMap<String, M2Model>();
|
||||
HashMap<String, M2Model> modelMap = new HashMap<>();
|
||||
if (alfrescoModelDir.exists() && alfrescoModelDir.isDirectory())
|
||||
{
|
||||
// A filter for XML files
|
||||
FileFilter filter = new FileFilter()
|
||||
{
|
||||
@Override
|
||||
public boolean accept(File pathname)
|
||||
{
|
||||
return pathname.isFile() && pathname.getName().endsWith(".xml");
|
||||
}
|
||||
|
||||
};
|
||||
FileFilter filter = pathname -> pathname.isFile() && pathname.getName().endsWith(".xml");
|
||||
// List XML files
|
||||
File[] files = alfrescoModelDir.listFiles(filter);
|
||||
if (files != null)
|
||||
@@ -289,24 +281,18 @@ public class ModelTracker extends AbstractTracker implements Tracker
|
||||
long start = System.nanoTime();
|
||||
|
||||
List<AlfrescoModelDiff> modelDiffs = client.getModelsDiff(coreName, this.infoSrv.getAlfrescoModels());
|
||||
HashMap<String, M2Model> modelMap = new HashMap<String, M2Model>();
|
||||
Map<String, M2Model> modelMap = new HashMap<>();
|
||||
|
||||
for (AlfrescoModelDiff modelDiff : modelDiffs)
|
||||
{
|
||||
switch (modelDiff.getType())
|
||||
{
|
||||
case CHANGED:
|
||||
AlfrescoModel changedModel = client.getModel(coreName, modelDiff.getModelName());
|
||||
for (M2Namespace namespace : changedModel.getModel().getNamespaces())
|
||||
{
|
||||
modelMap.put(namespace.getUri(), changedModel.getModel());
|
||||
}
|
||||
break;
|
||||
case NEW:
|
||||
AlfrescoModel newModel = client.getModel(coreName, modelDiff.getModelName());
|
||||
for (M2Namespace namespace : newModel.getModel().getNamespaces())
|
||||
AlfrescoModel model = client.getModel(coreName, modelDiff.getModelName());
|
||||
for (M2Namespace namespace : model.getModel().getNamespaces())
|
||||
{
|
||||
modelMap.put(namespace.getUri(), newModel.getModel());
|
||||
modelMap.put(namespace.getUri(), model.getModel());
|
||||
}
|
||||
break;
|
||||
case REMOVED:
|
||||
@@ -317,7 +303,7 @@ public class ModelTracker extends AbstractTracker implements Tracker
|
||||
}
|
||||
}
|
||||
|
||||
HashSet<String> loadedModels = new HashSet<String>();
|
||||
HashSet<String> loadedModels = new HashSet<>();
|
||||
for (M2Model model : modelMap.values())
|
||||
{
|
||||
loadModel(modelMap, loadedModels, model);
|
||||
@@ -367,7 +353,7 @@ public class ModelTracker extends AbstractTracker implements Tracker
|
||||
|
||||
trackerStats.addModelTime(end - start);
|
||||
|
||||
if (true == runPostModelLoadInit)
|
||||
if (runPostModelLoadInit)
|
||||
{
|
||||
for (Object key : props.keySet())
|
||||
{
|
||||
@@ -450,10 +436,7 @@ public class ModelTracker extends AbstractTracker implements Tracker
|
||||
{
|
||||
expandedQName = expandQNameImpl(qName);
|
||||
}
|
||||
// else if (AlfrescoSolrDataModel.nonDictionaryFields.get(qName) == null)
|
||||
// {
|
||||
// expandedQName = expandQNameImpl(qName);
|
||||
// }
|
||||
|
||||
return QName.createQName(expandedQName);
|
||||
|
||||
}
|
||||
@@ -495,10 +478,7 @@ public class ModelTracker extends AbstractTracker implements Tracker
|
||||
{
|
||||
expandedQName = expandQNameImpl(qName);
|
||||
}
|
||||
// else if (AlfrescoSolrDataModel.nonDictionaryFields.get(qName) == null)
|
||||
// {
|
||||
// expandedQName = expandQNameImpl(qName);
|
||||
// }
|
||||
|
||||
return expandedQName;
|
||||
|
||||
}
|
||||
@@ -513,26 +493,21 @@ public class ModelTracker extends AbstractTracker implements Tracker
|
||||
final String prefix = modelName.toPrefixString(this.infoSrv.getNamespaceDAO()).replace(":", ".") + ".";
|
||||
final String postFix = ".xml";
|
||||
|
||||
File[] toDelete = alfrescoModelDir.listFiles(new FileFilter()
|
||||
{
|
||||
@Override
|
||||
public boolean accept(File pathname)
|
||||
File[] toDelete = alfrescoModelDir.listFiles(pathname -> {
|
||||
if (pathname.isDirectory()) { return false; }
|
||||
String name = pathname.getName();
|
||||
if (!name.endsWith(postFix)) { return false; }
|
||||
if (!name.startsWith(prefix)) { return false; }
|
||||
// check is number between
|
||||
String checksum = name.substring(prefix.length(), name.length() - postFix.length());
|
||||
try
|
||||
{
|
||||
if (pathname.isDirectory()) { return false; }
|
||||
String name = pathname.getName();
|
||||
if (false == name.endsWith(postFix)) { return false; }
|
||||
if (false == name.startsWith(prefix)) { return false; }
|
||||
// check is number between
|
||||
String checksum = name.substring(prefix.length(), name.length() - postFix.length());
|
||||
try
|
||||
{
|
||||
Long.parseLong(checksum);
|
||||
return true;
|
||||
}
|
||||
catch (NumberFormatException nfe)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
Long.parseLong(checksum);
|
||||
return true;
|
||||
}
|
||||
catch (NumberFormatException nfe)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -548,7 +523,7 @@ public class ModelTracker extends AbstractTracker implements Tracker
|
||||
private void loadModel(Map<String, M2Model> modelMap, HashSet<String> loadedModels, M2Model model)
|
||||
{
|
||||
String modelName = model.getName();
|
||||
if (loadedModels.contains(modelName) == false)
|
||||
if (!loadedModels.contains(modelName))
|
||||
{
|
||||
for (M2Namespace importNamespace : model.getImports())
|
||||
{
|
||||
|
||||
+2
-2
@@ -280,7 +280,7 @@ public class AlfrescoSolrClusteringComponent extends SearchComponent implements
|
||||
list.add(doc);
|
||||
|
||||
if (ids != null) {
|
||||
ids.put(doc, new Integer(docid));
|
||||
ids.put(doc, Integer.valueOf(docid));
|
||||
}
|
||||
}
|
||||
return list;
|
||||
@@ -356,7 +356,7 @@ public class AlfrescoSolrClusteringComponent extends SearchComponent implements
|
||||
/**
|
||||
* @return Expose for tests.
|
||||
*/
|
||||
Map<String, SearchClusteringEngine> getSearchClusteringEngines() {
|
||||
Map<String, SearchClusteringEngine> getSearchClusteringEnginesView() {
|
||||
return searchClusteringEnginesView;
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -89,7 +89,7 @@ import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_VERSI
|
||||
@ThreadLeakLingering(linger = 5000)
|
||||
public abstract class AbstractAlfrescoDistributedTest extends SolrTestInitializer
|
||||
{
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
protected static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
protected String[] deadServers;
|
||||
protected static SolrResponsesComparator solrComparator = new SolrResponsesComparator();
|
||||
|
||||
+55
-60
@@ -18,6 +18,17 @@
|
||||
*/
|
||||
package org.alfresco.solr.tracker;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
import static java.util.stream.IntStream.range;
|
||||
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getNode;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
|
||||
|
||||
import org.alfresco.model.ContentModel;
|
||||
import org.alfresco.repo.index.shard.ShardMethodEnum;
|
||||
import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
|
||||
@@ -31,9 +42,9 @@ import org.alfresco.solr.client.Node;
|
||||
import org.alfresco.solr.client.NodeMetaData;
|
||||
import org.alfresco.solr.client.StringPropertyValue;
|
||||
import org.alfresco.solr.client.Transaction;
|
||||
import org.alfresco.util.CachingDateFormat;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.junit.AfterClass;
|
||||
@@ -49,40 +60,23 @@ import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.TimeZone;
|
||||
|
||||
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getNode;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
|
||||
import static org.alfresco.solr.AlfrescoSolrUtils.list;
|
||||
|
||||
/**
|
||||
* @author Joel
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
|
||||
@SolrTestCaseJ4.SuppressSSL
|
||||
@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"})
|
||||
public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest
|
||||
{
|
||||
@BeforeClass
|
||||
private static void initData() throws Throwable
|
||||
@SuppressWarnings("unused")
|
||||
public static void initData() throws Throwable
|
||||
{
|
||||
initSolrServers(5, "DistributedDateMonthAlfrescoSolrTrackerTest", getShardMethod());
|
||||
initSolrServers(5, DistributedDateMonthAlfrescoSolrTrackerTest.class.getSimpleName(), getShardMethod());
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
private static void destroyData() throws Throwable
|
||||
@SuppressWarnings("unused")
|
||||
public static void destroyData()
|
||||
{
|
||||
dismissSolrServers();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testDateMonth() throws Exception
|
||||
{
|
||||
@@ -91,58 +85,57 @@ public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfresc
|
||||
int numAcls = 25;
|
||||
AclChangeSet bulkAclChangeSet = getAclChangeSet(numAcls);
|
||||
|
||||
List<Acl> bulkAcls = new ArrayList();
|
||||
List<AclReaders> bulkAclReaders = new ArrayList();
|
||||
List<Acl> bulkAcls = new ArrayList<>();
|
||||
List<AclReaders> bulkAclReaders = new ArrayList<>();
|
||||
|
||||
|
||||
for (int i = 0; i < numAcls; i++) {
|
||||
for (int i = 0; i < numAcls; i++)
|
||||
{
|
||||
Acl bulkAcl = getAcl(bulkAclChangeSet);
|
||||
bulkAcls.add(bulkAcl);
|
||||
bulkAclReaders.add(getAclReaders(bulkAclChangeSet,
|
||||
bulkAcl,
|
||||
list("joel" + bulkAcl.getId()),
|
||||
list("phil" + bulkAcl.getId()),
|
||||
singletonList("joel" + bulkAcl.getId()),
|
||||
singletonList("phil" + bulkAcl.getId()),
|
||||
null));
|
||||
}
|
||||
|
||||
indexAclChangeSet(bulkAclChangeSet,
|
||||
bulkAcls,
|
||||
bulkAclReaders);
|
||||
indexAclChangeSet(bulkAclChangeSet, bulkAcls, bulkAclReaders);
|
||||
|
||||
int numNodes = 1000;
|
||||
List<Node> nodes = new ArrayList();
|
||||
List<NodeMetaData> nodeMetaDatas = new ArrayList();
|
||||
List<Node> nodes = new ArrayList<>();
|
||||
List<NodeMetaData> nodeMetaData = new ArrayList<>();
|
||||
|
||||
Transaction bigTxn = getTransaction(0, numNodes);
|
||||
|
||||
Date[] dates = new Date[5];
|
||||
Calendar calendar = new GregorianCalendar();
|
||||
calendar.setTimeZone(TimeZone.getTimeZone("UTC"));
|
||||
|
||||
Calendar cal = new GregorianCalendar();
|
||||
cal.setTimeZone(TimeZone.getTimeZone("UTC"));
|
||||
for (int i = 0; i < dates.length; i++) {
|
||||
|
||||
cal.set(1980, i, 21);
|
||||
dates[i] = cal.getTime();
|
||||
}
|
||||
Date[] dates = range(0, 5)
|
||||
.peek(index -> calendar.set(1980, index, 21))
|
||||
.mapToObj(index -> calendar.getTime())
|
||||
.toArray(Date[]::new);
|
||||
|
||||
int[] counts = new int[dates.length];
|
||||
|
||||
for (int i = 0; i < numNodes; i++) {
|
||||
for (int i = 0; i < numNodes; i++)
|
||||
{
|
||||
int aclIndex = i % numAcls;
|
||||
int dateIndex = i % dates.length;
|
||||
String dateString = DefaultTypeConverter.INSTANCE.convert(String.class, dates[dateIndex]);
|
||||
|
||||
counts[dateIndex]++;
|
||||
|
||||
Node node = getNode(bigTxn, bulkAcls.get(aclIndex), Node.SolrApiNodeStatus.UPDATED);
|
||||
node.setShardPropertyValue(dateString);
|
||||
nodes.add(node);
|
||||
NodeMetaData nodeMetaData = getNodeMetaData(node, bigTxn, bulkAcls.get(aclIndex), "mike", null, false);
|
||||
nodeMetaData.getProperties().put(ContentModel.PROP_CREATED,
|
||||
new StringPropertyValue(dateString));
|
||||
|
||||
nodeMetaDatas.add(nodeMetaData);
|
||||
NodeMetaData metadata = getNodeMetaData(node, bigTxn, bulkAcls.get(aclIndex), "mike", null, false);
|
||||
metadata.getProperties().put(ContentModel.PROP_CREATED, new StringPropertyValue(dateString));
|
||||
|
||||
nodeMetaData.add(metadata);
|
||||
}
|
||||
|
||||
indexTransaction(bigTxn, nodes, nodeMetaDatas);
|
||||
indexTransaction(bigTxn, nodes, nodeMetaData);
|
||||
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), numNodes, 100000);
|
||||
waitForDocCountAllCores(new TermQuery(new Term(FIELD_DOC_TYPE, SolrInformationServer.DOC_TYPE_ACL)), numAcls, 100000);
|
||||
|
||||
@@ -150,9 +143,7 @@ public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfresc
|
||||
AlfrescoSolrDataModel.FieldInstance fieldInstance = fieldInstanceList.get(0);
|
||||
String fieldName = fieldInstance.getField();
|
||||
|
||||
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
|
||||
format.setTimeZone(TimeZone.getTimeZone("GMT"));
|
||||
SimpleDateFormat format = CachingDateFormat.getSolrDatetimeFormatWithoutMsecs();
|
||||
for (int i = 0; i < dates.length; i++)
|
||||
{
|
||||
String startDate = format.format(dates[i]);
|
||||
@@ -161,32 +152,36 @@ public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfresc
|
||||
gcal.add(Calendar.SECOND, 1);
|
||||
String endDate = format.format(gcal.getTime());
|
||||
|
||||
SolrQuery solrQuery = new SolrQuery("{!lucene}" + escapeQueryChars(fieldName) +
|
||||
":[" + escapeQueryChars(startDate) + " TO " + escapeQueryChars(endDate) + " } " );
|
||||
SolrQuery solrQuery =
|
||||
new SolrQuery("{!lucene}" + escapeQueryChars(fieldName) + ":[" + escapeQueryChars(startDate) + " TO " + escapeQueryChars(endDate) + " } " );
|
||||
|
||||
assertCountAndColocation(solrQuery, counts[i]);
|
||||
assertShardSequence(i, solrQuery, counts[i]);
|
||||
}
|
||||
|
||||
nodes.clear();
|
||||
nodeMetaDatas.clear();
|
||||
nodeMetaData.clear();
|
||||
|
||||
Transaction bigTxn1 = getTransaction(0, numNodes);
|
||||
|
||||
for (int i = 0; i < numNodes; i++) {
|
||||
for (int i = 0; i < numNodes; i++)
|
||||
{
|
||||
int aclIndex = i % numAcls;
|
||||
Node node = getNode(bigTxn1, bulkAcls.get(aclIndex), Node.SolrApiNodeStatus.UPDATED);
|
||||
nodes.add(node);
|
||||
NodeMetaData nodeMetaData = getNodeMetaData(node, bigTxn1, bulkAcls.get(aclIndex), "mike", null, false);
|
||||
nodeMetaDatas.add(nodeMetaData);
|
||||
|
||||
NodeMetaData metadata = getNodeMetaData(node, bigTxn1, bulkAcls.get(aclIndex), "mike", null, false);
|
||||
nodeMetaData.add(metadata);
|
||||
}
|
||||
|
||||
indexTransaction(bigTxn1, nodes, nodeMetaDatas);
|
||||
indexTransaction(bigTxn1, nodes, nodeMetaData);
|
||||
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), numNodes*2, 100000);
|
||||
|
||||
//There are 5 shards. We should expect roughly 20% of the nodes on each shard
|
||||
assertNodesPerShardGreaterThan((int)((numNodes*2)*.17));
|
||||
}
|
||||
|
||||
protected static Properties getShardMethod()
|
||||
private static Properties getShardMethod()
|
||||
{
|
||||
Properties prop = new Properties();
|
||||
prop.put("shard.method", ShardMethodEnum.DATE.toString());
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<parent>
|
||||
<groupId>org.alfresco</groupId>
|
||||
<artifactId>alfresco-search-parent</artifactId>
|
||||
<version>1.4.0-SNAPSHOT</version>
|
||||
<version>1.1.0</version>
|
||||
</parent>
|
||||
|
||||
<distributionManagement>
|
||||
@@ -22,7 +22,7 @@
|
||||
</distributionManagement>
|
||||
|
||||
<properties>
|
||||
<dependency.alfresco-data-model.version>8.45</dependency.alfresco-data-model.version>
|
||||
<dependency.alfresco-data-model.version>8.46</dependency.alfresco-data-model.version>
|
||||
<dependency.jackson.version>2.10.0.pr1</dependency.jackson.version>
|
||||
</properties>
|
||||
|
||||
@@ -73,7 +73,7 @@
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>1.7.26</version>
|
||||
<version>1.7.27</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
+19
-4
@@ -26,6 +26,8 @@
|
||||
|
||||
package org.alfresco.solr;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
@@ -64,8 +66,21 @@ public class TrackerState
|
||||
private volatile boolean checkedLastAclTransactionTime = false;
|
||||
private volatile boolean checkedLastTransactionTime = false;
|
||||
|
||||
private volatile boolean check = false;
|
||||
private volatile int trackerCycles;
|
||||
private volatile boolean check = false;
|
||||
// Handle Thread Safe operations
|
||||
private volatile TrackerCyclesInteger trackerCycles;
|
||||
class TrackerCyclesInteger
|
||||
{
|
||||
private AtomicInteger value = new AtomicInteger(0);
|
||||
private void increase()
|
||||
{
|
||||
value.incrementAndGet();
|
||||
}
|
||||
private int getValue()
|
||||
{
|
||||
return value.get();
|
||||
}
|
||||
}
|
||||
private long timeToStopIndexing;
|
||||
|
||||
private long lastGoodChangeSetCommitTimeInIndex;
|
||||
@@ -237,13 +252,13 @@ public class TrackerState
|
||||
|
||||
public int getTrackerCycles()
|
||||
{
|
||||
return this.trackerCycles;
|
||||
return this.trackerCycles.getValue();
|
||||
}
|
||||
|
||||
public synchronized void incrementTrackerCycles()
|
||||
{
|
||||
log.debug("incrementTrackerCycles from :" + trackerCycles);
|
||||
this.trackerCycles++;
|
||||
this.trackerCycles.increase();
|
||||
log.debug("incremented TrackerCycles to :" + trackerCycles);
|
||||
}
|
||||
|
||||
|
||||
+9
-12
@@ -38,6 +38,7 @@ import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
|
||||
import org.alfresco.error.AlfrescoRuntimeException;
|
||||
@@ -741,7 +742,7 @@ public class SOLRAPIClient
|
||||
String localeStr = o.has("locale") && !o.isNull("locale") ? o.getString("locale") : null;
|
||||
Locale locale = (o.has("locale") && !o.isNull("locale") ? deserializer.deserializeValue(Locale.class, localeStr) : null);
|
||||
|
||||
Long size = o.has("size") && !o.isNull("size") ? o.getLong("size") : null;
|
||||
long size = o.has("size") && !o.isNull("size") ? o.getLong("size") : 0;
|
||||
|
||||
String encoding = o.has("encoding") && !o.isNull("encoding") ? o.getString("encoding") : null;
|
||||
String mimetype = o.has("mimetype") && !o.isNull("mimetype") ? o.getString("mimetype") : null;
|
||||
@@ -1247,17 +1248,13 @@ public class SOLRAPIClient
|
||||
this.namespaceDAO = namespaceDAO;
|
||||
|
||||
// add all default converters to this converter
|
||||
// TODO find a better way of doing this
|
||||
Map<Class<?>, Map<Class<?>, Converter<?,?>>> converters = DefaultTypeConverter.INSTANCE.getConverters();
|
||||
for(Class source : converters.keySet())
|
||||
{
|
||||
Map<Class<?>, Converter<?,?>> converters1 = converters.get(source);
|
||||
for(Class dest : converters1.keySet())
|
||||
{
|
||||
Converter<?,?> converter = converters1.get(dest);
|
||||
instance.addConverter(source, dest, converter);
|
||||
}
|
||||
}
|
||||
for (Entry<Class<?>, Map<Class<?>, Converter<?, ?>>> source : DefaultTypeConverter.INSTANCE.getConverters().entrySet())
|
||||
{
|
||||
for (Entry<Class<?>, Converter<?, ?>> dest : source.getValue().entrySet())
|
||||
{
|
||||
instance.addConverter((Class) source.getKey(), (Class) dest.getKey(), dest.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
// dates
|
||||
instance.addConverter(String.class, Date.class, new TypeConverter.Converter<String, Date>()
|
||||
|
||||
+12
-9
@@ -29,9 +29,12 @@ import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
import org.alfresco.solr.InformationServerCollectionProvider;
|
||||
import org.alfresco.solr.adapters.ISimpleOrderedMap;
|
||||
import org.alfresco.util.Pair;
|
||||
@@ -283,11 +286,10 @@ public class TrackerStats
|
||||
map.add("StdDev", getStandardDeviation());
|
||||
if (incdludeDetail)
|
||||
{
|
||||
for (String key : copies.keySet())
|
||||
{
|
||||
IncrementalStats value = copies.get(key);
|
||||
map.add(key, value.getNamedList(includeHist, includeValues));
|
||||
}
|
||||
for (Entry<String, IncrementalStats> copy : copies.entrySet())
|
||||
{
|
||||
map.add(copy.getKey(), copy.getValue().getNamedList(includeHist, includeValues));
|
||||
}
|
||||
}
|
||||
|
||||
return map;
|
||||
@@ -382,6 +384,7 @@ public class TrackerStats
|
||||
|
||||
}
|
||||
|
||||
@NotThreadSafe
|
||||
public static class IncrementalStats
|
||||
{
|
||||
Date start = new Date();
|
||||
@@ -769,7 +772,7 @@ public class TrackerStats
|
||||
{
|
||||
IncrementalStats copy = new IncrementalStats(this.scale, this.buckets, this.server);
|
||||
copy.start = this.start;
|
||||
copy.max = this.max;
|
||||
copy.max = this.getMax();
|
||||
copy.min = this.min;
|
||||
copy.moments[0] = this.moments[0];
|
||||
copy.moments[1] = this.moments[1];
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
<parent>
|
||||
<groupId>org.alfresco</groupId>
|
||||
<artifactId>alfresco-search-parent</artifactId>
|
||||
<version>1.4.0-SNAPSHOT</version>
|
||||
<version>1.1.0</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<dependencies>
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
ALFRESCO_TAG=5.2.5-SNAPSHOT
|
||||
SEARCH_TAG=latest
|
||||
ALFRESCO_TAG=5.2.6-RC2
|
||||
SHARE_TAG=5.2.6-RC2
|
||||
SEARCH_TAG=latest
|
||||
POSTGRES_TAG=9.4
|
||||
|
||||
@@ -1,33 +1,59 @@
|
||||
version: '3'
|
||||
# Using version 2 as 3 does not support resource constraint options (cpu_*, mem_* limits) for non swarm mode in Compose
|
||||
version: "2"
|
||||
|
||||
services:
|
||||
platform:
|
||||
image: docker-internal.alfresco.com/platform:${ALFRESCO_TAG}
|
||||
environment:
|
||||
JAVA_TOOL_OPTIONS: "
|
||||
-Dsolr.host=search
|
||||
-Dsolr.port=8983
|
||||
-Dsolr.secureComms=none
|
||||
-Dsolr.base.url=/solr
|
||||
-Dindex.subsystem.name=solr6
|
||||
"
|
||||
ports:
|
||||
- "7203:7203" #JMX connect via service:jmx:rmi:///jndi/rmi://localhost:7203/jmxrmi
|
||||
- "5005:5005" #Java debugging
|
||||
- "8081:8080" #Browser port for Alfresco
|
||||
- "8082:8080" #Browser port for Share
|
||||
- "8084:8080" #Browser port for API Explorer
|
||||
search:
|
||||
image: quay.io/alfresco/search-services:${SEARCH_TAG}
|
||||
environment:
|
||||
#Solr needs to know how to register itself with Alfresco
|
||||
SOLR_ALFRESCO_HOST: platform
|
||||
SOLR_ALFRESCO_PORT: 8080
|
||||
#Alfresco needs to know how to call solr
|
||||
SOLR_SOLR_HOST: search
|
||||
SOLR_SOLR_PORT: 8983
|
||||
#Create the default alfresco and archive cores
|
||||
SOLR_CREATE_ALFRESCO_DEFAULTS: alfresco,archive
|
||||
#HTTP by default
|
||||
ALFRESCO_SECURE_COMMS: none
|
||||
ports:
|
||||
- "8083:8983" #Browser port for Solr
|
||||
alfresco:
|
||||
image: quay.io/alfresco/alfresco-content-repository-52:${ALFRESCO_TAG}
|
||||
mem_limit: 2g
|
||||
environment:
|
||||
MEM_LIMIT: 2200m
|
||||
JAVA_TOOL_OPTIONS: "
|
||||
-Dsolr.host=search
|
||||
-Dsolr.port=8983
|
||||
-Dsolr.secureComms=none
|
||||
-Dsolr.base.url=/solr
|
||||
-Dindex.subsystem.name=solr6
|
||||
"
|
||||
ports:
|
||||
- 7203:7203 #JMX connect via service:jmx:rmi:///jndi/rmi://localhost:7203/jmxrmi
|
||||
- 5005:5005 #Java debugging
|
||||
- 8081:8080 #Browser port for Alfresco
|
||||
|
||||
share:
|
||||
image: quay.io/alfresco/alfresco-share-52:${SHARE_TAG}
|
||||
mem_limit: 1g
|
||||
environment:
|
||||
MEM_LIMIT: 1200m
|
||||
ports:
|
||||
- 8082:8080
|
||||
|
||||
search:
|
||||
image: quay.io/alfresco/search-services:${SEARCH_TAG}
|
||||
environment:
|
||||
#Solr needs to know how to register itself with Alfresco
|
||||
SOLR_ALFRESCO_HOST: alfresco
|
||||
SOLR_ALFRESCO_PORT: 8080
|
||||
#Alfresco needs to know how to call solr
|
||||
SOLR_SOLR_HOST: search
|
||||
SOLR_SOLR_PORT: 8983
|
||||
#Create the default alfresco and archive cores
|
||||
SOLR_CREATE_ALFRESCO_DEFAULTS: alfresco,archive
|
||||
#HTTP by default
|
||||
ALFRESCO_SECURE_COMMS: none
|
||||
ports:
|
||||
- 8083:8983
|
||||
|
||||
libreoffice:
|
||||
image: xcgd/libreoffice
|
||||
mem_limit: 1g
|
||||
|
||||
postgres:
|
||||
image: postgres:${POSTGRES_TAG}
|
||||
mem_limit: 512m
|
||||
environment:
|
||||
- POSTGRES_PASSWORD=alfresco
|
||||
- POSTGRES_USER=alfresco
|
||||
- POSTGRES_DB=alfresco
|
||||
command: postgres -c max_connections=300 -c log_min_messages=LOG
|
||||
ports:
|
||||
- 5432:5432
|
||||
|
||||
@@ -4,13 +4,13 @@
|
||||
<parent>
|
||||
<groupId>org.alfresco</groupId>
|
||||
<artifactId>alfresco-search-and-insight-parent</artifactId>
|
||||
<version>1.4.0-SNAPSHOT</version>
|
||||
<version>1.1.0</version>
|
||||
</parent>
|
||||
<!-- The groupId and version are required by the maven pom extractor plugin on Bamboo - more details in this issue:
|
||||
https://bitbucket.org/dehringer/bamboo-maven-pom-extractor-plugin/issues/18/groupid-not-populated-if-using-parent-pom -->
|
||||
<groupId>org.alfresco</groupId>
|
||||
<artifactId>alfresco-search-parent</artifactId>
|
||||
<version>1.4.0-SNAPSHOT</version>
|
||||
<version>1.1.0</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>Alfresco Solr Search parent</name>
|
||||
<modules>
|
||||
|
||||
Reference in New Issue
Block a user