From d42245773fa1ff2983dec9000550122c17208e1b Mon Sep 17 00:00:00 2001
From: ahind <andy.hind@alfresco.com>
Date: Thu, 24 Nov 2016 13:14:47 +0000
Subject: [PATCH] SEARCH-253   Tracking errors may not be recorded in the index
 and recovered.

---
 .../org/alfresco/solr/tracker/AbstractTracker.java |  2 +-
 .../solr/tracker/AbstractWorkerRunnable.java       | 14 +++++++++++++-
 .../java/org/alfresco/solr/tracker/AclTracker.java |  6 ++++++
 .../org/alfresco/solr/tracker/CascadeTracker.java  |  6 ++++++
 .../org/alfresco/solr/tracker/ContentTracker.java  |  6 ++++++
 .../org/alfresco/solr/tracker/MetadataTracker.java |  6 ++++++
 6 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java
index 43e2ff6d6..adaad4e44 100644
--- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java
+++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java
@@ -68,7 +68,7 @@ public abstract class AbstractTracker implements Tracker
     protected String shardMethod;
     protected boolean transformContent;
     protected String shardTemplate;
-    protected boolean rollback;
+    protected volatile boolean rollback;
 
     
     /*
diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java
index e7d2e001d..30f228ebe 100644
--- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java
+++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java
@@ -18,9 +18,13 @@
  */
 package org.alfresco.solr.tracker;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 abstract class AbstractWorkerRunnable implements Runnable
 {
+	protected final static Logger log = LoggerFactory.getLogger(AbstractWorkerRunnable.class);
+	
     QueueHandler queueHandler;
     
     public AbstractWorkerRunnable(QueueHandler qh)
@@ -35,20 +39,28 @@ abstract class AbstractWorkerRunnable implements Runnable
     @Override
     public void run()
     {
+    	boolean failed = true;
         try
         {
             doWork();
+            failed = false;
         }
         catch (Exception e)
         {
-            e.printStackTrace();
+        	log.warn("Index tracking batch hit an unrecoverable error ", e);
         }
         finally
         {
             // Triple check that we get the queue state right
             queueHandler.removeFromQueueAndProdHead(this);
+            if(failed)
+            {
+            	onFail();
+            }
         }
     }
     
     abstract protected void doWork() throws Exception;
+    
+    abstract protected void onFail();
 }
diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java
index 66e9e3cdd..9b50db4f7 100644
--- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java
+++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java
@@ -853,6 +853,12 @@ public class AclTracker extends AbstractTracker
             }
         }
         
+        @Override
+        protected void onFail()
+        {
+        	setRollback(true);
+        }
+        
         private List<Acl> filterAcls(List<Acl> acls)
         {
             ArrayList<Acl> filteredList = new ArrayList(acls.size());
diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java
index bde12bda7..3dac07080 100644
--- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java
+++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java
@@ -116,6 +116,12 @@ public class CascadeTracker extends AbstractTracker implements Tracker
         {
             this.infoServer.cascadeNodes(nodes, true);
         }
+        
+        @Override
+        protected void onFail()
+        {
+        	setRollback(true);
+        }
     }
 
     public void invalidateState() {
diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java
index 7154c71e6..eba54c27b 100644
--- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java
+++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java
@@ -156,5 +156,11 @@ public class ContentTracker extends AbstractTracker implements Tracker
             //System.out.println("################ Update doc:"+doc.dbId);
             this.infoServer.updateContentToIndexAndCache(doc.dbId, doc.tenant);
         }
+        
+        @Override
+        protected void onFail()
+        {
+        	// Will redo if not persisted
+        }
     }
 }
diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java
index b78831dd4..305408062 100644
--- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java
+++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java
@@ -821,6 +821,12 @@ public class MetadataTracker extends AbstractTracker implements Tracker
             }
         }
         
+        @Override
+        protected void onFail()
+        {
+        	setRollback(true);
+        }
+        
         private List<Node> filterNodes(List<Node> nodes)
         {
             ArrayList<Node> filteredList = new ArrayList<Node>(nodes.size());