Merge branch 'features/SEARCH-1620_DockerTemplates' into 'master'

Added SOLR Replication (Master/Slave) for Community and Enterprise

See merge request search_discovery/SearchAnalyticsE2ETest!58

e2e-test/generator-alfresco-docker-compose/README.md

@@ -6,6 +6,7 @@ This project generates a collection of Docker Compose Templates to test Reposito
 * Plain HTTP communications
 * TLS/SSL Mutual Authentication communications
 * Sharding (dynamic)
+* Replication (master/slave)
 
 ## Project structure
 
@@ -19,6 +20,8 @@ generators/app/templates/
 │   └── Dockerfile
 ├── docker-compose-ce.yml
 ├── docker-compose-ee.yml
+├── replication-none
+│   └── Dockerfile
 ├── search-https
 │   └── Dockerfile
 ├── sharding-https
@@ -41,6 +44,7 @@ generators/app/templates/
 * `zeppelin-https` includes a Dockerfile template to start Zeppelin with SSL
 * `sharding-none` includes a Dockerfile template for dynamic Sharding in Plain HTTP
 * `sharding-https` includes a Dockerfile template for dynamic Sharding with SSL
+* `replication-none` includes a Dockerfile template for Replication in Master/Slave mode
 * `keystores` includes every truststore and keystore required for SSL configuration
 
 
@@ -64,10 +68,13 @@ $ yo alfresco-docker-compose
 
 When using Community, Plain HTTP or TLS/SSL Mutual Auth can be selected.
 
+Additionally, if Plain HTTP is selected, SOLR Replication can be added.
+
 ```
 ? Which Alfresco version do you want to use? 6.1
 ? Would you like to use Alfresco enterprise or community? community
 ? Would you like to use http or https? http
+? Would you like to use a SOLR Cluster (2 nodes in master-slave)? Yes
 ```
 
 ## Enterprise
@@ -75,9 +82,10 @@ When using Community, Plain HTTP or TLS/SSL Mutual Auth can be selected.
 When using Enterprise, some different options can be combined:
 
 * Plain HTTP (http) or TLS/SSL Mutual Auth (https)
+* Use SOLR Replication in Master/Slave mode (only for http)
 * Insight Engine, as Search Services is selected by default
 * Deploy Zeppelin app to use JDBC Connector to SOLR
-* Use dynamic Sharding with 2 SOLR nodes pre-configured
+* Use dynamic Sharding with 2 SOLR nodes pre-configured (only when not using SOLR Replication)
 
 ```
 ? Which Alfresco version do you want to use? 6.1
@@ -88,6 +96,25 @@ When using Enterprise, some different options can be combined:
 ? Would you like to use dynamic Sharding (2 SOLR nodes)? Yes
 ```
 
+## Passing parameters from command line
+
+Default values for options can be specified in the command line, using a `--name=value` pattern. When an options is specified in the command line, the question is not prompted to the user, so you can generate a Docker Compose template with no user interaction.
+
+```
+$ yo alfresco-docker-compose --acsVersion=6.1 --alfrescoVersion=community --httpMode=http --clustering=true
+```
+
+**Parameter names reference**
+
+`--acsVersion`: currently only accepting 6.1
+`--alfrescoVersion`: community or enterprise
+`--httpMode`: http or https
+`--clustering`: true or false
+`--insightEngine`: true or false
+`--zeppelin`: true or false
+`--sharding`: true or false
+
+
 ## Using Docker Compose
 
 Once the files have been generated, just start Docker Compose.
@@ -112,6 +139,11 @@ http://localhost:8082/alfresco
 
 http://localhost:8083/solr
 
+When using SOLR Replication, additionally
+
+http://localhost:8084/solr
+
+
 SSL
 
 http://localhost:8080/share
@@ -134,6 +166,11 @@ http://localhost:8083/solr
 
 http://localhost:9090/zeppelin
 
+When using SOLR Replication, additionally
+
+http://localhost:8084/solr
+
+
 SSL
 
 http://localhost:8080/share

e2e-test/generator-alfresco-docker-compose/generators/app/index.js

@@ -9,6 +9,7 @@ var banner = require('./banner')
  * - Plain HTTP communications
  * - TLS/SSL Mutual Authentication communications
  * - Sharding (dynamic)
+ * - Clustering (master-slave)
 */
 module.exports = class extends Generator {
 
@@ -40,10 +41,19 @@ module.exports = class extends Generator {
         choices: [ "http", "https" ],
         default: 'http'
       },
+      {
+        when: function (response) {
+          return response.httpMode == 'http' || commandProps['httpMode'] == 'http';
+        },
+        type: 'confirm',
+        name: 'clustering',
+        message: 'Would you like to use a SOLR Cluster (2 nodes in master-slave)?',
+        default: false
+      },
       // Enterprise only options
       {
         when: function (response) {
-          return response.alfrescoVersion == 'enterprise';
+          return response.alfrescoVersion == 'enterprise' || commandProps['alfrescoVersion'] == 'enterprise';
         },
         type: 'confirm',
         name: 'insightEngine',
@@ -52,8 +62,8 @@ module.exports = class extends Generator {
       },
       {
         when: function (response) {
-          return response.alfrescoVersion == 'enterprise' &&
-                 response.insightEngine;
+          return (response.alfrescoVersion == 'enterprise' || commandProps['alfrescoVersion'] == 'enterprise') &&
+                 (response.insightEngine || commandProps['insightEngine']);
         },
         type: 'confirm',
         name: 'zeppelin',
@@ -62,7 +72,8 @@ module.exports = class extends Generator {
       },
       {
         when: function (response) {
-          return response.alfrescoVersion == 'enterprise';
+          return (response.alfrescoVersion == 'enterprise' || commandProps['alfrescoVersion'] == 'enterprise') &&
+                 (!response.clustering || !commandProps['clustering']);
         },
         type: 'confirm',
         name: 'sharding',
@@ -71,9 +82,24 @@ module.exports = class extends Generator {
       }
     ];
 
-    return this.prompt(prompts).then(props => {
+    // Read options from command line parameters
+    const filteredPrompts = [];
+    const commandProps = new Map();
+    prompts.forEach(function prompts(prompt) {
+      const option = this.options[prompt.name];
+      if (option === undefined) {
+        filteredPrompts.push(prompt);
+      } else {      
+        commandProps[prompt.name] = normalize(option, prompt); 
+      }
+    }, this);
+
+    // Prompt only for parameters not passed by command line
+    return this.prompt(filteredPrompts).then(props => {
       this.props = props;
+      Object.assign(props, commandProps);
     });
+
   }
 
   // Generate boilerplate from "templates" folder
@@ -86,7 +112,9 @@ module.exports = class extends Generator {
         this.destinationPath('docker-compose.yml'),
         { httpMode: this.props.httpMode,
           secureComms: (this.props.httpMode == 'http' ? 'none' : 'https'),
-          alfrescoPort: (this.props.httpMode == 'http' ? '8080' : '8443')
+          alfrescoPort: (this.props.httpMode == 'http' ? '8080' : '8443'),
+          clustering: (this.props.clustering ? "true" : "false"),
+          searchSolrHost: (this.props.clustering ? "solr6slave" : "solr6")
         }
       );
     
@@ -112,7 +140,9 @@ module.exports = class extends Generator {
             "alfresco-search-services"
           ),
           zeppelin: (this.props.zeppelin ? "true" : "false"),
-          sharding: (this.props.sharding ? "true" : "false")
+          sharding: (this.props.sharding ? "true" : "false"),
+          clustering: (this.props.clustering ? "true" : "false"),
+          searchSolrHost: (this.props.clustering ? "solr6slave" : "solr6")
         }
       );
     }
@@ -168,6 +198,24 @@ module.exports = class extends Generator {
       }
     }
 
+    // Copy replication configuration
+    if (this.props.clustering) {
+      this.fs.copyTpl(
+        this.templatePath(this.props.acsVersion + '/replication-none'),
+        this.destinationPath('replication-none'),
+        {
+          searchImage: (this.props.insightEngine ?
+          "quay.io/alfresco/insight-engine" :
+          "alfresco/alfresco-search-services"
+          ),
+          searchPath: (this.props.insightEngine ?
+            "alfresco-insight-engine" :
+            "alfresco-search-services"
+          )
+        }
+      )
+    }
+
     // Copy sharding configuration
     if (this.props.sharding) {
       if (this.props.httpMode == 'https') {
@@ -198,3 +246,20 @@ module.exports = class extends Generator {
   }
 
 };
+
+// Convert parameter string value to boolean value
+function normalize(option, prompt) {
+  
+  if (prompt.type === 'confirm' && typeof option === 'string') {
+    let lc = option.toLowerCase();
+    if (lc === 'true' || lc === 'false') {
+      return (lc === 'true');
+    } else {
+      return option;
+    }
+  }
+
+  return option;
+
+}
+

e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml

@@ -27,7 +27,7 @@ services:
                 -Ddb.username=alfresco
                 -Ddb.password=alfresco
                 -Ddb.url=jdbc:postgresql://postgres:5432/alfresco
-                -Dsolr.host=solr6
+                -Dsolr.host=<%=searchSolrHost%>
                 -Dsolr.port.ssl=8983 
                 -Dsolr.secureComms=<%=secureComms%>
                 -Dsolr.base.url=/solr
@@ -47,6 +47,7 @@ services:
         volumes:
             - ./keystores/alfresco:/usr/local/tomcat/alf_data/keystore <% } %>
 
+    <% if (clustering == 'false') { %>
     solr6: <% if (httpMode == 'http') { %>
         image: alfresco/alfresco-search-services:${SEARCH_CE_TAG} <% } else { %>
         build:
@@ -81,6 +82,54 @@ services:
             - 8083:8983 <% if (httpMode == 'https') { %>
         volumes:
             - ./keystores/solr:/opt/alfresco-search-services/keystore <% } %>
+    <% } %>
+
+    <% if (clustering == 'true') { %>
+    solr6master:
+        build:
+          context: ./replication-<%=secureComms%>
+          args:
+            SEARCH_TAG: ${SEARCH_CE_TAG}
+            SOLR_HOSTNAME: solr6master
+            ENABLE_MASTER: "true"
+            ENABLE_SLAVE: "false"
+        mem_limit: 2g
+        environment:
+            #Solr needs to know how to register itself with Alfresco
+            SOLR_ALFRESCO_HOST: "alfresco"
+            SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>"
+            #Alfresco needs to know how to call solr
+            SOLR_SOLR_HOST: "solr6master"
+            SOLR_SOLR_PORT: "8983"
+            #Create the default alfresco and archive cores
+            SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive"
+            SOLR_JAVA_MEM: "-Xms2g -Xmx2g"
+        ports:
+            - 8083:8983
+
+    solr6slave:
+        build:
+          context: ./replication-<%=secureComms%>
+          args:
+            SEARCH_TAG: ${SEARCH_CE_TAG}
+            SOLR_HOSTNAME: solr6slave
+            ENABLE_MASTER: "false"
+            ENABLE_SLAVE: "true"
+            MASTER_HOST: solr6master
+        mem_limit: 2g
+        environment:
+            #Solr needs to know how to register itself with Alfresco
+            SOLR_ALFRESCO_HOST: "alfresco"
+            SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>"
+            #Alfresco needs to know how to call solr
+            SOLR_SOLR_HOST: "solr6slave"
+            SOLR_SOLR_PORT: "8983"
+            #Create the default alfresco and archive cores
+            SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive"
+            SOLR_JAVA_MEM: "-Xms2g -Xmx2g"
+        ports:
+            - 8084:8983
+    <% } %>
 
     share:
         image: alfresco/alfresco-share:${SHARE_TAG}

e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ee.yml

@@ -26,7 +26,7 @@ services:
                 -Ddb.username=alfresco
                 -Ddb.password=alfresco
                 -Ddb.url=jdbc:postgresql://postgres:5432/alfresco
-                -Dsolr.host=solr6
+                -Dsolr.host=<%=searchSolrHost%>
                 -Dsolr.port.ssl=8983
                 -Dsolr.secureComms=<%=secureComms%>
                 -Dsolr.base.url=/solr <% if (sharding == 'true') { %>
@@ -55,7 +55,7 @@ services:
         volumes:
             - ./keystores/alfresco:/usr/local/tomcat/alf_data/keystore <% } %>
 
-    <% if (sharding == 'false') { %>
+    <% if (sharding == 'false' && clustering == 'false') { %>
     solr6: <% if (httpMode == 'http') { %>
         image: <%=searchImage%>:${<%=searchTag%>} <% } else { %>
         build:
@@ -138,6 +138,53 @@ services:
             - 8084:8983 #Browser port
     <% } %>
 
+    <% if (clustering == 'true') { %>
+    solr6master:
+        build:
+          context: ./replication-<%=secureComms%>
+          args:
+            SEARCH_TAG: ${<%=searchTag%>}
+            SOLR_HOSTNAME: solr6master
+            ENABLE_MASTER: "true"
+            ENABLE_SLAVE: "false"
+        mem_limit: 2g
+        environment:
+            #Solr needs to know how to register itself with Alfresco
+            SOLR_ALFRESCO_HOST: "alfresco"
+            SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>"
+            #Alfresco needs to know how to call solr
+            SOLR_SOLR_HOST: "solr6master"
+            SOLR_SOLR_PORT: "8983"
+            #Create the default alfresco and archive cores
+            SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive"
+            SOLR_JAVA_MEM: "-Xms2g -Xmx2g"
+        ports:
+            - 8083:8983
+
+    solr6slave:
+        build:
+          context: ./replication-<%=secureComms%>
+          args:
+            SEARCH_TAG: ${<%=searchTag%>}
+            SOLR_HOSTNAME: solr6slave
+            ENABLE_MASTER: "false"
+            ENABLE_SLAVE: "true"
+            MASTER_HOST: solr6master
+        mem_limit: 2g
+        environment:
+            #Solr needs to know how to register itself with Alfresco
+            SOLR_ALFRESCO_HOST: "alfresco"
+            SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>"
+            #Alfresco needs to know how to call solr
+            SOLR_SOLR_HOST: "solr6slave"
+            SOLR_SOLR_PORT: "8983"
+            #Create the default alfresco and archive cores
+            SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive"
+            SOLR_JAVA_MEM: "-Xms2g -Xmx2g"
+        ports:
+            - 8084:8983
+    <% } %>
+
     <% if (zeppelin == 'true') { %>
     zeppelin: <% if (httpMode == 'http') { %>
         image: quay.io/alfresco/insight-zeppelin:${ZEPPELIN_TAG} <% } else { %>

e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/replication-none/Dockerfile

@@ -0,0 +1,42 @@
+ARG SEARCH_TAG
+FROM <%=searchImage%>:${SEARCH_TAG}
+
+ARG SOLR_HOSTNAME
+ARG ENABLE_MASTER
+ARG ENABLE_SLAVE
+ARG MASTER_HOST
+ENV SOLR_HOSTNAME $SOLR_HOSTNAME
+ENV ENABLE_MASTER $ENABLE_MASTER
+ENV ENABLE_SLAVE $ENABLE_SLAVE
+ENV MASTER_HOST $MASTER_HOST
+
+# Configure SOLR cores to run in HTTP mode from template
+RUN sed -i '/^bash.*/i sed -i "'"s/alfresco.secureComms=https/alfresco.secureComms=none/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \
+    ${DIST_DIR}/solr/bin/search_config_setup.sh
+
+# Configure Alfresco Service Name
+RUN sed -i '/^bash.*/i sed -i "'"s/alfresco.host=localhost/alfresco.host=alfresco/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \
+    ${DIST_DIR}/solr/bin/search_config_setup.sh
+
+# Set Hostname for this Node
+RUN sed -i '/^bash.*/i sed -i "'"s/solr.host=localhost/solr.host=${SOLR_HOSTNAME}/g"'" ${DIST_DIR}/solrhome/conf/shared.properties\n' \
+    ${DIST_DIR}/solr/bin/search_config_setup.sh
+
+# Set Master / Slave configuration for this Node
+RUN sed -i '/^bash.*/i echo "\nenable.master=${ENABLE_MASTER}\nenable.slave=${ENABLE_SLAVE}" >> ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \
+${DIST_DIR}/solr/bin/search_config_setup.sh
+
+RUN if [ "$ENABLE_MASTER" = "true" ] ; then \
+sed -i "/^bash.*/i sed -i '/^\\\\\s*<requestHandler name=\"\\\\/replication\".*/a \
+    <lst name=\"master\">\
+      <str name=\"replicateAfter\">commit</str>\
+      <str name=\"replicateAfter\">startup</str>\
+      <str name=\"confFiles\">schema.xml,stopwords.txt</str>\
+    </lst>' ${DIST_DIR}/solrhome/templates/rerank/conf/solrconfig.xml\n" ${DIST_DIR}/solr/bin/search_config_setup.sh; \
+else \
+sed -i "/^bash.*/i sed -i '/^\\\\\s*<requestHandler name=\"\\\\/replication\".*/a \
+    <lst name=\"slave\">\
+      <str name=\"masterUrl\">http://${MASTER_HOST}:8983/solr/alfresco</str>\
+      <str name=\"pollInterval\">00:00:60</str>\
+    </lst>' ${DIST_DIR}/solrhome/templates/rerank/conf/solrconfig.xml\n" ${DIST_DIR}/solr/bin/search_config_setup.sh; \
+fi