SEARCH-253 Tracking errors may not be recorded in the index and recovered.

search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java

@@ -68,7 +68,7 @@ public abstract class AbstractTracker implements Tracker
     protected String shardMethod;
     protected boolean transformContent;
     protected String shardTemplate;
-    protected boolean rollback;
+    protected volatile boolean rollback;
 
     
     /*

search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java

@@ -18,9 +18,13 @@
  */
 package org.alfresco.solr.tracker;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 abstract class AbstractWorkerRunnable implements Runnable
 {
+	protected final static Logger log = LoggerFactory.getLogger(AbstractWorkerRunnable.class);
+	
     QueueHandler queueHandler;
     
     public AbstractWorkerRunnable(QueueHandler qh)
@@ -35,20 +39,28 @@ abstract class AbstractWorkerRunnable implements Runnable
     @Override
     public void run()
     {
+    	boolean failed = true;
         try
         {
             doWork();
+            failed = false;
         }
         catch (Exception e)
         {
-            e.printStackTrace();
+        	log.warn("Index tracking batch hit an unrecoverable error ", e);
         }
         finally
         {
             // Triple check that we get the queue state right
             queueHandler.removeFromQueueAndProdHead(this);
+            if(failed)
+            {
+            	onFail();
+            }
         }
     }
     
     abstract protected void doWork() throws Exception;
+    
+    abstract protected void onFail();
 }

search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java

@@ -853,6 +853,12 @@ public class AclTracker extends AbstractTracker
             }
         }
         
+        @Override
+        protected void onFail()
+        {
+        	setRollback(true);
+        }
+        
         private List<Acl> filterAcls(List<Acl> acls)
         {
             ArrayList<Acl> filteredList = new ArrayList(acls.size());

search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java

@@ -116,6 +116,12 @@ public class CascadeTracker extends AbstractTracker implements Tracker
         {
             this.infoServer.cascadeNodes(nodes, true);
         }
+        
+        @Override
+        protected void onFail()
+        {
+        	setRollback(true);
+        }
     }
 
     public void invalidateState() {

search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java

@@ -156,5 +156,11 @@ public class ContentTracker extends AbstractTracker implements Tracker
             //System.out.println("################ Update doc:"+doc.dbId);
             this.infoServer.updateContentToIndexAndCache(doc.dbId, doc.tenant);
         }
+        
+        @Override
+        protected void onFail()
+        {
+        	// Will redo if not persisted
+        }
     }
 }

search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java

@@ -821,6 +821,12 @@ public class MetadataTracker extends AbstractTracker implements Tracker
             }
         }
         
+        @Override
+        protected void onFail()
+        {
+        	setRollback(true);
+        }
+        
         private List<Node> filterNodes(List<Node> nodes)
         {
             ArrayList<Node> filteredList = new ArrayList<Node>(nodes.size());