Merge branch 'feature/add-benchmark' into 'master'

SEARCH-1505: feature/add benchmark See merge request search_discovery/SearchAnalyticsE2ETest!19
2025-09-10 14:11:25 +00:00 · 2019-03-20 15:08:40 +00:00
parent 87b595207c 95b5ca3beb
commit ea2b66816d
7 changed files with 292 additions and 0 deletions
--- a/e2e-test/qa/benchmark/Makefile
+++ b/e2e-test/qa/benchmark/Makefile
@@ -0,0 +1,31 @@
+include ../Makefile
+
+ALFRESCO_URL?=http://localhost:8081
+export ALFRESCO_URL
+
+update: 
+	# this will override existing files that you have already in this folder
+	# use it only if you want to get the latest version from github
+	@wget -O .env https://raw.githubusercontent.com/Alfresco/alfresco-bm-manager/master/server/docker-compose/.env
+	@wget -O docker-compose-drivers.yml https://raw.githubusercontent.com/Alfresco/alfresco-bm-manager/master/server/docker-compose/docker-compose-drivers.yml
+	@wget -O docker-compose-drivers.yml https://raw.githubusercontent.com/Alfresco/alfresco-bm-manager/master/server/docker-compose/docker-compose-drivers.yml
+	@wget -O docker-compose-manager.yml https://raw.githubusercontent.com/Alfresco/alfresco-bm-manager/master/server/docker-compose/docker-compose-manager.yml
+
+start-manager: ## 1 - start BM Manager
+	$(dc) -f docker-compose-manager.yml up -d && \
+	$(helpers)/wait-service-to-start.sh http://localhost:9080/alfresco-bm-manager
+
+start-drivers: ## 2 - start BM Drivers
+	$(dc) -f docker-compose-drivers.yml pull && docker-compose -f docker-compose-manager.yml -f docker-compose-drivers.yml up -d
+
+start-all: start-manager start-drivers ## 3 - start BM Manager and Driver
+
+clean: ## clean BM Manager and Drivers
+	$(dc) -f docker-compose-manager.yml down || true && $(dc) -f docker-compose-drivers.yml down || true
+
+benchmark-prepare: ## 1 - prepare data with users/sites for benchmark
+	./prepare-users.sh
+	./prepare-data.sh
+
+benchmark-run: ## 2 - here we will actually execute the RestAPI for Search
+	./benchmark-search.sh
--- a/e2e-test/qa/benchmark/README.md
+++ b/e2e-test/qa/benchmark/README.md
@@ -0,0 +1,41 @@
+## About
+
+Starting automatically Benchmark infrastructure:
+* [BM Manager](https://github.com/Alfresco/alfresco-bm-manager) 
+* [BM Drivers](https://github.com/Alfresco/alfresco-bm-manager/tree/master/docs/bm-driver)
+  * running create [users](https://github.com/Alfresco/alfresco-bm-load-users)
+  * running create [data](https://github.com/Alfresco/alfresco-bm-load-data)
+  * running [rest test](https://github.com/Alfresco/alfresco-bm-rest-api)
+
+## Prerequisites
+* ACS deployed in AWS (EKS or Docker)
+* Benchmark knowledge (check the official documentation above)
+
+## How to use it?
+### a) start BM Manager and Drivers locally
+
+```shell
+$ make start-all
+```
+>check the official documentation how to use the benchmark
+
+### b) prepare data for benchmarking
+> we need to create users/sites and load some content in order to tests the search performance
+
+```shell
+$ make benchmark-prepare
+```
+>this will call some shell scripts
+
+### c) run the actual benchmark tests
+>you can tweak the benchmark tests as you want in the shell script executed by this task
+```shell
+$ make start-all
+```
+
+### `!` Bamboo Build
+We defined a sad benchmark build: [SAD-BEN](https://bamboo.alfresco.com/bamboo/browse/SAD-BEN) that will execute the steps above automatically for you.
+
+If you run it customized, you can override the `ALFRESCO_URL` variable to point to a deployed ACS instance (default to: "http://ec2-34-245-148-19.eu-west-1.compute.amazonaws.com:8081") 
+
+At the end of the tests you will have access to Excel reports (the actual benchmark results)
--- a/e2e-test/qa/benchmark/benchmark-search.sh
+++ b/e2e-test/qa/benchmark/benchmark-search.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+REQ_HOST="localhost:9080"
+REQ_TESTS_API="http://$REQ_HOST/alfresco-bm-manager/api/v1/tests"
+
+echo -ne "Create Search Test\n"
+#Create Search Test
+TEST_NAME="SearchTest"
+curl -s $REQ_TESTS_API -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"name":"'$TEST_NAME'","description":"Users","release":"alfresco-bm-rest-api-3.0.1-SNAPSHOT","schema":"12"}'
+
+echo -ne "Set MongoDB Search\n"
+#Set Mongo DB
+IP=$(hostname -I | awk '{print $1}')
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/mongo.test.host" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$IP':27017"}'
+
+echo "Set Alfresco related attributes Search"
+#Set Alfresco related attributes
+#ALF_SERVER=${bamboo.alfresco.url}
+#ALF_SERVER="ec2-34-245-148-19.eu-west-1.compute.amazonaws.com"
+ALF_SERVER=$(echo $ALFRESCO_URL | cut -d '/' -f3 | cut -d ':' -f1)
+ALF_PORT=$(echo $ALFRESCO_URL | cut -d ':' -f3)
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/alfresco.server" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$ALF_SERVER'"}'
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/alfresco.url" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"http://'$ALF_SERVER:$ALF_PORT'/"}'
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/alfresco.port" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$ALF_PORT'"}'
+
+#Set Scenario Weightings
+echo -ne "Set Scenario Weightings\n"
+#curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/weight.4.read" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":0}'
+#curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/weight.4.read" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":0}'
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/weight.read" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":0}'
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/weight.write" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":0}'
+
+
+echo -ne "Create test instance Search\n"
+#Create test instance
+INSTANCE_NAME="RunTestSearch1"
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/runs" -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"name":"'$INSTANCE_NAME'"}'
+
+echo -ne "Start the test Search\n"
+#Start the test
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/runs/"$INSTANCE_NAME"/schedule" -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"scheduled":0}'
+
+completed=-1
+# Add 0 to the completed value to ensure it's a number.
+while [ $((completed+0)) -eq -1 ]
+do
+    sleep 10
+    status=`curl -s $REQ_TESTS_API"/"$TEST_NAME"/runs" -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8'`
+    #log "Status: $status"
+    # Update the completed timestamp value (which will be -1 if the test hasn't finished yet).
+    completed=`echo $status | sed 's|^.*"completed" : \([^ ]*\) .*$|\1|g'`
+    if [ $((completed+0)) -eq -1 ]
+    then
+        # Also check the "stopped" value if it hasn't completed.
+        completed=`echo $status | sed 's|^.*"stopped" : \([^ ]*\) .*$|\1|g'`
+    fi
+    #log "Progress: "`echo $status | sed 's|^.*"progress" : \([^ ]*\) .*$|\1|g'`
+done
+
+#log "Output the CSV results."
+curl -s -o search_results.xlsx -O -J $REQ_TESTS_API"/"$TEST_NAME"/runs/"$INSTANCE_NAME"/results/xlsx"
--- a/e2e-test/qa/benchmark/docker-compose-drivers.yml
+++ b/e2e-test/qa/benchmark/docker-compose-drivers.yml
@@ -0,0 +1,27 @@
+version: "3"
+# This is the docker compose for Alfresco Benchmark drivers
+
+services:
+    alfresco-bm-load-users:
+        image: quay.io/alfresco/alfresco-bm-load-users:${ALFRESCO_BM_LOAD_USERS_TAG}
+        environment:
+            JAVA_OPTS : "
+                -Dmongo.config.host=mongo:${MONGO_PORT}
+                -Dserver.port=9082
+                "
+
+    alfresco-bm-load-data:
+        image: quay.io/alfresco/alfresco-bm-load-data:${ALFRESCO_BM_LOAD_DATA_TAG}
+        environment:
+            JAVA_OPTS : "
+                -Dmongo.config.host=mongo:${MONGO_PORT}
+                -Dserver.port=9083
+                "
+
+    alfresco-bm-rest-api:
+        image: quay.io/alfresco/alfresco-bm-rest-api:${ALFRESCO_BM_REST_API_TAG}
+        environment:
+            JAVA_OPTS : "
+                -Dmongo.config.host=mongo:${MONGO_PORT}
+                -Dserver.port=9084
+                "
--- a/e2e-test/qa/benchmark/docker-compose-manager.yml
+++ b/e2e-test/qa/benchmark/docker-compose-manager.yml
@@ -0,0 +1,17 @@
+version: "3"
+# This is the docker compose for Alfresco Benchmark Manager
+
+services:
+    alfresco-bm-manager:
+        image: quay.io/alfresco/alfresco-bm-manager:${ALFRESCO_BM_MANAGER_TAG}
+        environment:
+            JAVA_OPTS : "
+                -Dmongo.config.host=mongo
+                "
+        ports:
+            - 9080:9080
+
+    mongo:
+        image: mongo:${MONGO_TAG}
+        ports:
+            - ${MONGO_PORT}:27017
--- a/e2e-test/qa/benchmark/prepare-data.sh
+++ b/e2e-test/qa/benchmark/prepare-data.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -ex
+REQ_HOST="localhost:9080"
+REQ_TESTS_API="http://$REQ_HOST/alfresco-bm-manager/api/v1/tests"
+TEST_NAME="DataTest"
+
+echo -ne "Create Test Data\n"
+#Create Test
+curl -s $REQ_TESTS_API -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"name":"'$TEST_NAME'","description":"Data","release":"alfresco-bm-load-data-3.0.1-SNAPSHOT","schema":"11"}'
+
+echo -ne "Set MongoDB Data\n"
+#Set Mongo DB
+IP=$(hostname -I | awk '{print $1}')
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/mongo.test.host" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$IP':27017"}'
+
+echo -ne "Set Alfresco related attributes Data\n"
+#Set Alfresco related attributes
+#ALF_SERVER=${bamboo.alfresco.url}
+#ALF_SERVER="ec2-34-245-148-19.eu-west-1.compute.amazonaws.com"
+ALF_SERVER=$(echo $ALFRESCO_URL | cut -d '/' -f3 | cut -d ':' -f1)
+ALF_PORT=$(echo $ALFRESCO_URL | cut -d ':' -f3)
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/alfresco.server" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$ALF_SERVER'"}'
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/alfresco.url" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$ALFRESCO_URL'/"}'
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/alfresco.port" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$ALF_PORT'"}'
+
+#Set Site attribute
+SITE_NAME=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 4 | head -n 1)-Site-%s-%05d
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/load.siteFormat" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$SITE_NAME'"}'
+
+echo -ne "Create test instance Data\n"
+#Create test instance
+INSTANCE_NAME="RunTestData1"
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/runs" -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"name":"'$INSTANCE_NAME'"}'
+
+echo -ne "Start the test Data\n"
+#Start the test
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/runs/"$INSTANCE_NAME"/schedule" -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"scheduled":0}'
+
+completed=-1
+# Add 0 to the completed value to ensure it's a number.
+while [ $((completed+0)) -eq -1 ]
+do
+    sleep 10
+    status=`curl -s $REQ_TESTS_API"/"$TEST_NAME"/runs" -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8'`
+    #log "Status: $status"
+    # Update the completed timestamp value (which will be -1 if the test hasn't finished yet).
+    completed=`echo $status | sed 's|^.*"completed" : \([^ ]*\) .*$|\1|g'`
+    if [ $((completed+0)) -eq -1 ]
+    then
+        # Also check the "stopped" value if it hasn't completed.
+        completed=`echo $status | sed 's|^.*"stopped" : \([^ ]*\) .*$|\1|g'`
+    fi
+    #log "Progress: "`echo $status | sed 's|^.*"progress" : \([^ ]*\) .*$|\1|g'`
+done
+
+curl -s -o create_data_results.xlsx -O -J $REQ_TESTS_API"/"$TEST_NAME"/runs/"$INSTANCE_NAME"/results/xlsx"
--- a/e2e-test/qa/benchmark/prepare-users.sh
+++ b/e2e-test/qa/benchmark/prepare-users.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -ex
+REQ_HOST="localhost:9080"
+REQ_TESTS_API="http://$REQ_HOST/alfresco-bm-manager/api/v1/tests"
+TEST_NAME="UserTest"
+
+
+echo -ne "Create Test\n"
+#Create Test
+curl -s $REQ_TESTS_API -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"name":"'$TEST_NAME'","description":"Users","release":"alfresco-bm-load-users-3.0.1-SNAPSHOT","schema":"9"}'
+
+echo -e "Set MongoDB\n"
+#Set Mongo DB
+IP=$(hostname -I | awk '{print $1}')
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/mongo.test.host" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$IP':27017"}'
+
+echo -e "Set Alfresco related attributes\n"
+#Set Alfresco related attributes
+#ALF_SERVER=${bamboo.alfresco.url}
+#ALF_SERVER="ec2-34-245-148-19.eu-west-1.compute.amazonaws.com"
+ALF_SERVER=$(echo $ALFRESCO_URL | cut -d '/' -f3 | cut -d ':' -f1)
+ALF_PORT=$(echo $ALFRESCO_URL | cut -d ':' -f3)
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/alfresco.server" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$ALF_SERVER'"}'
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/alfresco.url" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"http://'$ALF_SERVER:$ALF_PORT'/"}'
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/alfresco.port" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$ALF_PORT'"}'
+
+echo -e "Create Users\n"
+#Create Users
+NO_USERS="300"
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/props/user.numberOfUsers" -X PUT -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"value":"'$NO_USERS'"}'
+
+echo -e "Create test instance\n"
+#Create test instance
+INSTANCE_NAME="RunTestUser1"
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/runs" -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"name":"'$INSTANCE_NAME'"}'
+
+echo -r "Start the test\n"
+#Start the test
+curl -s $REQ_TESTS_API"/"$TEST_NAME"/runs/"$INSTANCE_NAME"/schedule" -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8' --data '{"version":0,"scheduled":0}'
+
+completed=-1
+# Add 0 to the completed value to ensure it's a number.
+while [ $((completed+0)) -eq -1 ]
+do
+    sleep 10
+    status=`curl -s $REQ_TESTS_API"/"$TEST_NAME"/runs" -H "'Host: '"$REQ_HOST"/'" -H 'Content-Type: application/json;charset=utf-8'`
+    #log "Status: $status"
+    # Update the completed timestamp value (which will be -1 if the test hasn't finished yet).
+    completed=`echo $status | sed 's|^.*"completed" : \([^ ]*\) .*$|\1|g'`
+    if [ $((completed+0)) -eq -1 ]
+    then
+        # Also check the "stopped" value if it hasn't completed.
+        completed=`echo $status | sed 's|^.*"stopped" : \([^ ]*\) .*$|\1|g'`
+    fi
+    #log "Progress: "`echo $status | sed 's|^.*"progress" : \([^ ]*\) .*$|\1|g'`
+done
+
+#log "Output the CSV results."
+curl -s -o create_users_results.xlsx -O -J $REQ_TESTS_API"/"$TEST_NAME"/runs/"$INSTANCE_NAME"/results/xlsx"