MOB-647 - activity feed cleaner improvement (now supports maxFeedSize in addition to maxAgeMins)

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@14119 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/activities/activities-feed-context.xml

@@ -18,7 +18,10 @@
    <bean id="feedCleaner" class="org.alfresco.repo.activities.feed.cleanup.FeedCleaner">
       <property name="feedDAO" ref="feedDAO"/>
       <property name="maxAgeMins">
-         <value>44640</value> <!-- 44640 mins = 31 days -->
+         <value>${activities.feed.max.age.mins}</value> <!-- max age in mins, eg. 44640 mins = 31 days -->
+      </property>
+      <property name="maxFeedSize">
+         <value>${activities.feed.max.size}</value> <!--  max entries per site feed or user feed (to nearest postDate) -->
       </property>
    </bean>
    

config/alfresco/ibatis/org.hibernate.dialect.Dialect/activities-common-SqlMap.xml

@@ -44,6 +44,16 @@
       <result property="lastModified" column="last_modified"/>
    </resultMap>
    
+   <!-- select feeds for cleaning -->
+   <select id="select.activity.feed.greater.than.max" parameterClass="int" resultClass="ActivityFeed">
+      <![CDATA[
+      select site_network as siteNetwork, feed_user_id as feedUserId, activity_format as activitySummaryFormat
+      from alf_activity_feed
+      group by site_network, feed_user_id, activity_format
+      having count(*) > #maxFeedSize#
+      ]]>
+   </select>
+   
    <!-- user feed - all sites - everyone -->
    <select id="select.activity.feed.for.feeduser" parameterClass="ActivityFeed" resultClass="ActivityFeed">
       <![CDATA[
@@ -135,6 +145,24 @@
       ]]>
    </delete>
    
+   <delete id="delete.activity.feed.for.site.entries.older.than.date" parameterClass="ActivityFeed">
+      <![CDATA[
+      delete from alf_activity_feed 
+      where post_date < #postDate#
+      and site_network = #siteNetwork#
+      and activity_format = #activitySummaryFormat#
+      ]]>
+   </delete>
+   
+   <delete id="delete.activity.feed.for.feeduser.entries.older.than.date" parameterClass="ActivityFeed">
+      <![CDATA[
+      delete from alf_activity_feed 
+      where post_date < #postDate#
+      and feed_user_id = #feedUserId#
+      and activity_format = #activitySummaryFormat#
+      ]]>
+   </delete>
+   
    <select id="select.activity.feedcontrols.for.user" parameterClass="FeedControl" resultClass="FeedControl">
       <![CDATA[
       select id as id, feed_user_id as feedUserId, site_network as siteNetwork, app_tool as appTool

config/alfresco/repository.properties

@@ -319,3 +319,7 @@ V2.1-A.fixes.to.schema=0
 
 # The default authentication chain
 authentication.chain=alfrescoNtlm1:alfrescoNtlm
+
+# Activity feed max size and max age (eg. 44640 mins = 31 days)
+activities.feed.max.size=100
+activities.feed.max.age.mins=44640

source/java/org/alfresco/repo/activities/feed/cleanup/FeedCleaner.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2005-2008 Alfresco Software Limited.
+ * Copyright (C) 2005-2009 Alfresco Software Limited.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -26,9 +26,10 @@ package org.alfresco.repo.activities.feed.cleanup;
 
 import java.sql.SQLException;
 import java.util.Date;
+import java.util.List;
 
-import org.alfresco.error.AlfrescoRuntimeException;
 import org.alfresco.repo.domain.activities.ActivityFeedDAO;
+import org.alfresco.repo.domain.activities.ActivityFeedEntity;
 import org.alfresco.util.PropertyCheck;
 import org.alfresco.util.VmShutdownListener;
 import org.apache.commons.logging.Log;
@@ -46,6 +47,8 @@ public class FeedCleaner
     
     private int maxAgeMins = 0;
     
+    private int maxFeedSize = -1; //unlimited
+    
     private ActivityFeedDAO feedDAO;
     
     public void setFeedDAO(ActivityFeedDAO feedDAO)
@@ -58,6 +61,12 @@ public class FeedCleaner
         this.maxAgeMins = mins;
     }
     
+    // note: this relates to user feed size (across all sites) or site feed size - for each format
+    public void setMaxFeedSize(int size)
+    {
+        this.maxFeedSize = size;
+    }
+    
     /**
      * Perform basic checks to ensure that the necessary dependencies were injected.
      */
@@ -65,29 +74,117 @@ public class FeedCleaner
     {
         PropertyCheck.mandatory(this, "feedDAO", feedDAO);
         
-        // check the max age
+        // check the max age and max feed size
-        if (maxAgeMins <= 0)
+        if ((maxAgeMins <= 0) && (maxFeedSize <= 0))
         {
-            throw new AlfrescoRuntimeException("Property 'maxAgeMins' must be greater than 0");
+            logger.warn("Neither maxAgeMins or maxFeedSize set - feeds will not be cleaned");
         }
     }
         
-    public void execute() throws JobExecutionException
+    public int execute() throws JobExecutionException
     {
         checkProperties();
+        
+        int maxAgeDeletedCount = 0;
+        int maxSizeDeletedCount = 0;
+        
         try
         {
+            if (maxAgeMins > 0)
+            {
+                // clean old entries based on maxAgeMins
+                
                 long nowTimeOffset = new Date().getTime();
                 long keepTimeOffset = nowTimeOffset - ((long)maxAgeMins*60000L); // millsecs = mins * 60 secs * 1000 msecs
                 Date keepDate = new Date(keepTimeOffset);
                 
-            // clean old entries   
+                maxAgeDeletedCount = feedDAO.deleteFeedEntries(keepDate);
-            int deletedCount = feedDAO.deleteFeedEntries(keepDate);
                 
+                if (maxAgeDeletedCount > 0)
+                {
                     if (logger.isDebugEnabled())
                     {
+                        logger.debug("Cleaned " + maxAgeDeletedCount + " entries (upto " + keepDate + ", max age " + maxAgeMins + " mins)");
+                    }
+                }
+                else
+                {
+                    if (logger.isTraceEnabled())
+                    {
+                        logger.trace("Cleaned " + maxAgeDeletedCount + " entries (upto " + keepDate + ", max age " + maxAgeMins + " mins)");
+                    }
+                }
+            }
             
-                logger.debug("Cleaned " + deletedCount + " entries (upto " + keepDate + ", max age " + maxAgeMins + " mins)");
+            if (maxFeedSize > 0)
+            {
+                // clean old entries based on maxFeedSize
+                
+                // return candidate feeds to clean - either site+format or user+format
+                List<ActivityFeedEntity> feeds = feedDAO.selectFeedsToClean(maxFeedSize);
+                
+                int feedCount = 0;
+                
+                for (ActivityFeedEntity feed : feeds)
+                {
+                    String siteId = feed.getSiteNetwork();
+                    String feedUserId = feed.getFeedUserId();
+                    String format = feed.getActivitySummaryFormat();
+                    
+                    List<ActivityFeedEntity> feedToClean;
+                    
+                    if ((feedUserId == null) || (feedUserId.length() == 0))
+                    {
+                        feedToClean = feedDAO.selectSiteFeedEntries(siteId, format);
+                    }
+                    else
+                    {
+                        feedToClean = feedDAO.selectUserFeedEntries(feedUserId, format, null, false, false);
+                    }
+                    
+                    if (feedToClean.size() > maxFeedSize)
+                    {
+                        Date oldestFeedEntry = feedToClean.get(maxFeedSize-1).getPostDate();
+                        
+                        int deletedCount = 0;
+                        
+                        if ((feedUserId == null) || (feedUserId.length() == 0))
+                        {
+                            deletedCount = feedDAO.deleteUserFeedEntries(feedUserId, format, oldestFeedEntry);
+                        }
+                        else
+                        {
+                            deletedCount = feedDAO.deleteSiteFeedEntries(siteId, format, oldestFeedEntry);
+                        }
+                        
+                        
+                        if (deletedCount > 0)
+                        {
+                            maxSizeDeletedCount = maxSizeDeletedCount + deletedCount;
+                            feedCount++;
+                            
+                            if (logger.isTraceEnabled())
+                            {
+                                logger.trace("Cleaned " + deletedCount + " entries for ["+feed.getSiteNetwork()+", "+feed.getFeedUserId()+", "+feed.getActivitySummaryFormat()+"] (upto " + oldestFeedEntry + ")");
+                            }
+                        }
+                    }
+                }
+                
+                if (maxSizeDeletedCount > 0)
+                {
+                    if (logger.isDebugEnabled())
+                    {
+                        logger.debug("Cleaned " + maxSizeDeletedCount + " entries across " + feedCount + " feeds (max feed size "+maxFeedSize+" entries)");
+                    }
+                }
+                else
+                {
+                    if (logger.isTraceEnabled())
+                    {
+                        logger.trace("Cleaned " + maxSizeDeletedCount + " entries across " + feedCount + " feeds (max feed size "+maxFeedSize+" entries)");
+                    }
+                }
             }
         }
         catch (SQLException e)
@@ -107,5 +204,7 @@ public class FeedCleaner
                 logger.error("Exception during cleanup of feeds", e);
             }
         }
+        
+        return (maxAgeDeletedCount + maxSizeDeletedCount);
     }
 }

source/java/org/alfresco/repo/activities/feed/cleanup/FeedCleanerTest.java

@@ -0,0 +1,315 @@
+/*
+ * Copyright (C) 2005-2009 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of 
+ * the GPL, you may redistribute this Program in connection with Free/Libre 
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's 
+ * FLOSS exception.  You should have recieved a copy of the text describing 
+ * the FLOSS exception, and it is also available here: 
+ * http://www.alfresco.com/legal/licensing"
+ */
+package org.alfresco.repo.activities.feed.cleanup;
+
+import java.util.Date;
+import java.util.concurrent.CountDownLatch;
+
+import junit.framework.TestCase;
+
+import org.alfresco.repo.domain.activities.ActivityFeedDAO;
+import org.alfresco.repo.domain.activities.ActivityFeedEntity;
+import org.alfresco.repo.security.authentication.AuthenticationUtil;
+import org.alfresco.service.cmr.activities.ActivityService;
+import org.alfresco.util.ApplicationContextHelper;
+import org.springframework.context.ConfigurableApplicationContext;
+
+/**
+ * @see org.alfresco.repo.activities.feed.cleanup.FeedCleaner
+ * 
+ * @author janv
+ */
+public class FeedCleanerTest extends TestCase
+{
+    private static ConfigurableApplicationContext ctx = ApplicationContextHelper.getApplicationContext();
+    
+    private ActivityFeedDAO feedDAO;
+    private FeedCleaner cleaner;
+    private ActivityService activityService;
+    
+    @Override
+    public void setUp() throws Exception
+    {
+        AuthenticationUtil.setRunAsUserSystem();
+        
+        activityService = (ActivityService) ctx.getBean("activityService");
+        
+        feedDAO = (ActivityFeedDAO) ctx.getBean("feedDAO");
+                
+        // construct the test cleaner
+        cleaner = new FeedCleaner();
+        cleaner.setFeedDAO(feedDAO);
+    }
+    
+    public void tearDown() throws Exception
+    {
+        // clean out any remaining feed entries (allows test to be re-runnable)
+        feedDAO.deleteFeedEntries(new Date(System.currentTimeMillis()+(120*1000L))); 
+        
+        AuthenticationUtil.clearCurrentSecurityContext();
+    }
+    
+    public void testSetup() throws Exception
+    {
+        // NOOP
+    }
+    
+    public void testMaxAge() throws Exception
+    {
+        cleaner.setMaxFeedSize(0);
+        
+        // insert site feed entries for "testSite1"
+        
+        ActivityFeedEntity feedEntry = new ActivityFeedEntity();
+        
+        feedEntry.setPostDate(new Date(System.currentTimeMillis()-(20*60*1000L))); // 20 mins ago
+        feedEntry.setActivitySummaryFormat("json");
+        feedEntry.setSiteNetwork("testSite1");
+        feedEntry.setActivityType("testActivityType");
+        feedEntry.setPostUserId("testUserA");
+        feedEntry.setFeedUserId("");
+        feedEntry.setFeedDate(new Date());
+        
+        feedDAO.insertFeedEntry(feedEntry);
+        
+        feedEntry = new ActivityFeedEntity();
+        
+        feedEntry.setPostDate(new Date()); // now
+        feedEntry.setActivitySummaryFormat("json");
+        feedEntry.setSiteNetwork("testSite1");
+        feedEntry.setActivityType("testActivityType");
+        feedEntry.setPostUserId("testUserA");
+        feedEntry.setFeedUserId("");
+        feedEntry.setFeedDate(new Date());
+        
+        // insert user feed entries for "testUserB"
+        
+        feedDAO.insertFeedEntry(feedEntry);
+        
+        feedEntry = new ActivityFeedEntity();
+        
+        feedEntry.setPostDate(new Date(System.currentTimeMillis()-(20*60*1000L))); // 20 mins ago
+        feedEntry.setActivitySummaryFormat("json");
+        feedEntry.setSiteNetwork("testSite2");
+        feedEntry.setActivityType("testActivityType");
+        feedEntry.setPostUserId("testUserA");
+        feedEntry.setFeedUserId("testUserB");
+        feedEntry.setFeedDate(new Date());
+        
+        feedDAO.insertFeedEntry(feedEntry);
+        
+        feedEntry = new ActivityFeedEntity();
+        
+        feedEntry.setPostDate(new Date()); // now
+        feedEntry.setActivitySummaryFormat("json");
+        feedEntry.setSiteNetwork("testSite3");
+        feedEntry.setActivityType("testActivityType");
+        feedEntry.setPostUserId("testUserA");
+        feedEntry.setFeedUserId("testUserB");
+        feedEntry.setFeedDate(new Date());
+        
+        feedDAO.insertFeedEntry(feedEntry);
+        
+        assertEquals(2, activityService.getSiteFeedEntries("testSite1", "json").size());
+        assertEquals(2, activityService.getUserFeedEntries("testUserB", "json", null).size());
+        
+        // fire the cleaner
+        cleaner.setMaxAgeMins(10);
+        cleaner.execute();
+        
+        assertEquals(1, activityService.getSiteFeedEntries("testSite1", "json").size());
+        assertEquals(1, activityService.getUserFeedEntries("testUserB", "json", null).size());
+    }
+    
+    public void testMaxSize() throws Exception
+    {
+        cleaner.setMaxAgeMins(0);
+        
+        // insert site feed entries for "testSite4"
+        
+        for (int i = 0; i < 10; i++)
+        {
+            ActivityFeedEntity feedEntry = new ActivityFeedEntity();
+            
+            feedEntry.setPostDate(new Date(System.currentTimeMillis()-(i*60*1000L)));
+            feedEntry.setActivitySummaryFormat("json");
+            feedEntry.setSiteNetwork("testSite4");
+            feedEntry.setActivityType("testActivityType");
+            feedEntry.setPostUserId("testUserC");
+            feedEntry.setFeedUserId("");
+            feedEntry.setFeedDate(new Date());
+            
+            feedDAO.insertFeedEntry(feedEntry);
+        }
+        
+        // insert user feed entries for user "testUserD"
+        
+        for (int i = 0; i < 10; i++)
+        {
+            ActivityFeedEntity feedEntry = new ActivityFeedEntity();
+            
+            feedEntry.setPostDate(new Date(System.currentTimeMillis()-(i*60*1000L)));
+            feedEntry.setActivitySummaryFormat("json");
+            feedEntry.setSiteNetwork("testSite5");
+            feedEntry.setActivityType("testActivityType");
+            feedEntry.setPostUserId("testUserA");
+            feedEntry.setFeedUserId("testUserD");
+            feedEntry.setFeedDate(new Date());
+            
+            feedDAO.insertFeedEntry(feedEntry);
+        }
+        
+        assertEquals(10, activityService.getSiteFeedEntries("testSite4", "json").size());
+        assertEquals(10, activityService.getUserFeedEntries("testUserD", "json", null).size());
+        
+        // fire the cleaner
+        cleaner.setMaxFeedSize(2);
+        cleaner.execute();
+        
+        assertEquals(2, activityService.getSiteFeedEntries("testSite4", "json").size());
+        assertEquals(2, activityService.getUserFeedEntries("testUserD", "json", null).size());
+        
+        Date sameTime = new Date();
+        
+        // insert site feed entries for "testSite6"
+        
+        for (int i = 0; i < 10; i++)
+        {
+            ActivityFeedEntity feedEntry = new ActivityFeedEntity();
+            
+            feedEntry.setPostDate(sameTime);
+            feedEntry.setActivitySummaryFormat("json");
+            feedEntry.setSiteNetwork("testSite6");
+            feedEntry.setActivityType("testActivityType");
+            feedEntry.setPostUserId("testUserE");
+            feedEntry.setFeedUserId("");
+            feedEntry.setFeedDate(new Date());
+            
+            feedDAO.insertFeedEntry(feedEntry);
+        }
+        
+        // insert user feed entries for user "testUserF"
+        
+        for (int i = 0; i < 10; i++)
+        {
+            ActivityFeedEntity feedEntry = new ActivityFeedEntity();
+            
+            feedEntry.setPostDate(sameTime);
+            feedEntry.setActivitySummaryFormat("json");
+            feedEntry.setSiteNetwork("testSite7");
+            feedEntry.setActivityType("testActivityType");
+            feedEntry.setPostUserId("testUserA");
+            feedEntry.setFeedUserId("testUserF");
+            feedEntry.setFeedDate(new Date());
+            
+            feedDAO.insertFeedEntry(feedEntry);
+        }
+        
+        assertEquals(10, activityService.getSiteFeedEntries("testSite6", "json").size());
+        assertEquals(10, activityService.getUserFeedEntries("testUserF", "json", null).size());
+        
+        // fire the cleaner
+        cleaner.setMaxFeedSize(2);
+        cleaner.execute();
+        
+        // note: no effect, since entries at max feed size have same time (eg. to nearest minute)
+        assertEquals(10, activityService.getSiteFeedEntries("testSite6", "json").size());
+        assertEquals(10, activityService.getUserFeedEntries("testUserF", "json", null).size());
+    }
+    
+    public void testConcurrentAccessAndRemoval() throws Exception
+    {
+        cleaner.setMaxAgeMins(1);
+        cleaner.setMaxFeedSize(1);
+        
+        final int typeCount = 3;
+        int threadCount = typeCount * 10;
+        
+        final CountDownLatch endLatch = new CountDownLatch(threadCount);
+        // Kick off the threads
+        for (int i = 0; i < threadCount; i++)
+        {
+            Thread thread = new Thread(""+i)
+            {
+                @Override
+                public void run()
+                {
+                    try
+                    {
+                        int type = new Integer(this.getName()) % typeCount;
+                        
+                        if (type == 0)
+                        {
+                            int insertCount = 10;
+                            
+                            // insert some entries
+                            for (int i = 0; i < insertCount; i++)
+                            {
+                                ActivityFeedEntity feedEntry = new ActivityFeedEntity();
+                                
+                                feedEntry.setPostDate(new Date(System.currentTimeMillis()-(i*60*1000L)));
+                                feedEntry.setActivitySummaryFormat("json");
+                                feedEntry.setSiteNetwork("testSite4");
+                                feedEntry.setActivityType("testActivityType");
+                                feedEntry.setPostUserId("testUserC");
+                                feedEntry.setFeedUserId("");
+                                feedEntry.setFeedDate(new Date());
+                                
+                                feedDAO.insertFeedEntry(feedEntry);
+                            }
+                            
+                            System.out.println("["+this.getName()+"] Inserted "+insertCount+" entries");
+                        }
+                       
+                        if (type == 1)
+                        {
+                            // query some entries
+                            int selectCount = activityService.getSiteFeedEntries("testSite4", "json").size();
+                        
+                            System.out.println("["+this.getName()+"] Selected "+selectCount+" entries");
+                        }
+                        
+                        if (type == 2)
+                        {
+                            // clean some entries
+                            int deleteCount = cleaner.execute();
+                            
+                            System.out.println("["+this.getName()+"] Deleted "+deleteCount+" entries");
+                        }
+                    }
+                    catch (Exception e)
+                    {
+                        fail(e.getMessage());
+                    }
+                    // Notify of completion
+                    endLatch.countDown();
+                }
+            };
+            thread.start();
+        }
+        // Wait for them all to be done
+        endLatch.await();
+    }
+}

source/java/org/alfresco/repo/domain/activities/ActivityFeedDAO.java

@@ -37,6 +37,12 @@ public interface ActivityFeedDAO extends ActivitiesDAO
     
     public int deleteFeedEntries(Date keepDate) throws SQLException;
     
+    public int deleteUserFeedEntries(String feedUserId, String format, Date keepDate) throws SQLException;
+    
+    public int deleteSiteFeedEntries(String siteId, String format, Date keepDate) throws SQLException;
+    
+    public List<ActivityFeedEntity> selectFeedsToClean(int maxFeedSize) throws SQLException;
+    
     public List<ActivityFeedEntity> selectUserFeedEntries(String feedUserId, String format, String siteId, boolean excludeThisUser, boolean excludeOtherUsers) throws SQLException;
     
     public List<ActivityFeedEntity> selectSiteFeedEntries(String siteUserId, String format) throws SQLException;

source/java/org/alfresco/repo/domain/activities/ibatis/ActivityFeedDAOImpl.java

@@ -46,6 +46,32 @@ public class ActivityFeedDAOImpl extends IBatisSqlMapper implements ActivityFeed
         return getSqlMapClient().delete("delete.activity.feed.entries.older.than.date", keepDate);
     }
     
+    public int deleteSiteFeedEntries(String siteId, String format, Date keepDate) throws SQLException
+    {
+        ActivityFeedEntity params = new ActivityFeedEntity();
+        params.setSiteNetwork(siteId);
+        params.setActivitySummaryFormat(format);
+        params.setPostDate(keepDate);
+        
+        return getSqlMapClient().delete("delete.activity.feed.for.site.entries.older.than.date", params);
+    }
+    
+    public int deleteUserFeedEntries(String feedUserId, String format, Date keepDate) throws SQLException
+    {
+        ActivityFeedEntity params = new ActivityFeedEntity();
+        params.setFeedUserId(feedUserId);
+        params.setActivitySummaryFormat(format);
+        params.setPostDate(keepDate);
+        
+        return getSqlMapClient().delete("delete.activity.feed.for.feeduser.entries.older.than.date", params);
+    }
+    
+    @SuppressWarnings("unchecked")
+    public List<ActivityFeedEntity> selectFeedsToClean(int maxFeedSize) throws SQLException
+    {
+        return (List<ActivityFeedEntity>)getSqlMapClient().queryForList("select.activity.feed.greater.than.max", maxFeedSize);
+    }
+    
     @SuppressWarnings("unchecked")
     public List<ActivityFeedEntity> selectUserFeedEntries(String feedUserId, String format, String siteId, boolean excludeThisUser, boolean excludeOtherUsers) throws SQLException
     {