MOB-647 - activity feed cleaner improvement (now supports maxFeedSize in addition to maxAgeMins)

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@14119 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Jan Vonka
2009-04-29 11:24:03 +00:00
parent 7cd1ccae96
commit eb0db2737a
7 changed files with 515 additions and 34 deletions

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2005-2008 Alfresco Software Limited.
* Copyright (C) 2005-2009 Alfresco Software Limited.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@@ -26,9 +26,10 @@ package org.alfresco.repo.activities.feed.cleanup;
import java.sql.SQLException;
import java.util.Date;
import java.util.List;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.repo.domain.activities.ActivityFeedDAO;
import org.alfresco.repo.domain.activities.ActivityFeedEntity;
import org.alfresco.util.PropertyCheck;
import org.alfresco.util.VmShutdownListener;
import org.apache.commons.logging.Log;
@@ -46,6 +47,8 @@ public class FeedCleaner
private int maxAgeMins = 0;
private int maxFeedSize = -1; //unlimited
private ActivityFeedDAO feedDAO;
public void setFeedDAO(ActivityFeedDAO feedDAO)
@@ -58,6 +61,12 @@ public class FeedCleaner
this.maxAgeMins = mins;
}
// note: this relates to user feed size (across all sites) or site feed size - for each format
public void setMaxFeedSize(int size)
{
this.maxFeedSize = size;
}
/**
* Perform basic checks to ensure that the necessary dependencies were injected.
*/
@@ -65,29 +74,117 @@ public class FeedCleaner
{
PropertyCheck.mandatory(this, "feedDAO", feedDAO);
// check the max age
if (maxAgeMins <= 0)
// check the max age and max feed size
if ((maxAgeMins <= 0) && (maxFeedSize <= 0))
{
throw new AlfrescoRuntimeException("Property 'maxAgeMins' must be greater than 0");
logger.warn("Neither maxAgeMins or maxFeedSize set - feeds will not be cleaned");
}
}
public void execute() throws JobExecutionException
public int execute() throws JobExecutionException
{
checkProperties();
int maxAgeDeletedCount = 0;
int maxSizeDeletedCount = 0;
try
{
long nowTimeOffset = new Date().getTime();
long keepTimeOffset = nowTimeOffset - ((long)maxAgeMins*60000L); // millsecs = mins * 60 secs * 1000 msecs
Date keepDate = new Date(keepTimeOffset);
// clean old entries
int deletedCount = feedDAO.deleteFeedEntries(keepDate);
if (logger.isDebugEnabled())
{
if (maxAgeMins > 0)
{
// clean old entries based on maxAgeMins
logger.debug("Cleaned " + deletedCount + " entries (upto " + keepDate + ", max age " + maxAgeMins + " mins)");
long nowTimeOffset = new Date().getTime();
long keepTimeOffset = nowTimeOffset - ((long)maxAgeMins*60000L); // millsecs = mins * 60 secs * 1000 msecs
Date keepDate = new Date(keepTimeOffset);
maxAgeDeletedCount = feedDAO.deleteFeedEntries(keepDate);
if (maxAgeDeletedCount > 0)
{
if (logger.isDebugEnabled())
{
logger.debug("Cleaned " + maxAgeDeletedCount + " entries (upto " + keepDate + ", max age " + maxAgeMins + " mins)");
}
}
else
{
if (logger.isTraceEnabled())
{
logger.trace("Cleaned " + maxAgeDeletedCount + " entries (upto " + keepDate + ", max age " + maxAgeMins + " mins)");
}
}
}
if (maxFeedSize > 0)
{
// clean old entries based on maxFeedSize
// return candidate feeds to clean - either site+format or user+format
List<ActivityFeedEntity> feeds = feedDAO.selectFeedsToClean(maxFeedSize);
int feedCount = 0;
for (ActivityFeedEntity feed : feeds)
{
String siteId = feed.getSiteNetwork();
String feedUserId = feed.getFeedUserId();
String format = feed.getActivitySummaryFormat();
List<ActivityFeedEntity> feedToClean;
if ((feedUserId == null) || (feedUserId.length() == 0))
{
feedToClean = feedDAO.selectSiteFeedEntries(siteId, format);
}
else
{
feedToClean = feedDAO.selectUserFeedEntries(feedUserId, format, null, false, false);
}
if (feedToClean.size() > maxFeedSize)
{
Date oldestFeedEntry = feedToClean.get(maxFeedSize-1).getPostDate();
int deletedCount = 0;
if ((feedUserId == null) || (feedUserId.length() == 0))
{
deletedCount = feedDAO.deleteUserFeedEntries(feedUserId, format, oldestFeedEntry);
}
else
{
deletedCount = feedDAO.deleteSiteFeedEntries(siteId, format, oldestFeedEntry);
}
if (deletedCount > 0)
{
maxSizeDeletedCount = maxSizeDeletedCount + deletedCount;
feedCount++;
if (logger.isTraceEnabled())
{
logger.trace("Cleaned " + deletedCount + " entries for ["+feed.getSiteNetwork()+", "+feed.getFeedUserId()+", "+feed.getActivitySummaryFormat()+"] (upto " + oldestFeedEntry + ")");
}
}
}
}
if (maxSizeDeletedCount > 0)
{
if (logger.isDebugEnabled())
{
logger.debug("Cleaned " + maxSizeDeletedCount + " entries across " + feedCount + " feeds (max feed size "+maxFeedSize+" entries)");
}
}
else
{
if (logger.isTraceEnabled())
{
logger.trace("Cleaned " + maxSizeDeletedCount + " entries across " + feedCount + " feeds (max feed size "+maxFeedSize+" entries)");
}
}
}
}
catch (SQLException e)
@@ -107,5 +204,7 @@ public class FeedCleaner
logger.error("Exception during cleanup of feeds", e);
}
}
return (maxAgeDeletedCount + maxSizeDeletedCount);
}
}

View File

@@ -0,0 +1,315 @@
/*
* Copyright (C) 2005-2009 Alfresco Software Limited.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* As a special exception to the terms and conditions of version 2.0 of
* the GPL, you may redistribute this Program in connection with Free/Libre
* and Open Source Software ("FLOSS") applications as described in Alfresco's
* FLOSS exception. You should have recieved a copy of the text describing
* the FLOSS exception, and it is also available here:
* http://www.alfresco.com/legal/licensing"
*/
package org.alfresco.repo.activities.feed.cleanup;
import java.util.Date;
import java.util.concurrent.CountDownLatch;
import junit.framework.TestCase;
import org.alfresco.repo.domain.activities.ActivityFeedDAO;
import org.alfresco.repo.domain.activities.ActivityFeedEntity;
import org.alfresco.repo.security.authentication.AuthenticationUtil;
import org.alfresco.service.cmr.activities.ActivityService;
import org.alfresco.util.ApplicationContextHelper;
import org.springframework.context.ConfigurableApplicationContext;
/**
* @see org.alfresco.repo.activities.feed.cleanup.FeedCleaner
*
* @author janv
*/
public class FeedCleanerTest extends TestCase
{
private static ConfigurableApplicationContext ctx = ApplicationContextHelper.getApplicationContext();
private ActivityFeedDAO feedDAO;
private FeedCleaner cleaner;
private ActivityService activityService;
@Override
public void setUp() throws Exception
{
AuthenticationUtil.setRunAsUserSystem();
activityService = (ActivityService) ctx.getBean("activityService");
feedDAO = (ActivityFeedDAO) ctx.getBean("feedDAO");
// construct the test cleaner
cleaner = new FeedCleaner();
cleaner.setFeedDAO(feedDAO);
}
public void tearDown() throws Exception
{
// clean out any remaining feed entries (allows test to be re-runnable)
feedDAO.deleteFeedEntries(new Date(System.currentTimeMillis()+(120*1000L)));
AuthenticationUtil.clearCurrentSecurityContext();
}
public void testSetup() throws Exception
{
// NOOP
}
public void testMaxAge() throws Exception
{
cleaner.setMaxFeedSize(0);
// insert site feed entries for "testSite1"
ActivityFeedEntity feedEntry = new ActivityFeedEntity();
feedEntry.setPostDate(new Date(System.currentTimeMillis()-(20*60*1000L))); // 20 mins ago
feedEntry.setActivitySummaryFormat("json");
feedEntry.setSiteNetwork("testSite1");
feedEntry.setActivityType("testActivityType");
feedEntry.setPostUserId("testUserA");
feedEntry.setFeedUserId("");
feedEntry.setFeedDate(new Date());
feedDAO.insertFeedEntry(feedEntry);
feedEntry = new ActivityFeedEntity();
feedEntry.setPostDate(new Date()); // now
feedEntry.setActivitySummaryFormat("json");
feedEntry.setSiteNetwork("testSite1");
feedEntry.setActivityType("testActivityType");
feedEntry.setPostUserId("testUserA");
feedEntry.setFeedUserId("");
feedEntry.setFeedDate(new Date());
// insert user feed entries for "testUserB"
feedDAO.insertFeedEntry(feedEntry);
feedEntry = new ActivityFeedEntity();
feedEntry.setPostDate(new Date(System.currentTimeMillis()-(20*60*1000L))); // 20 mins ago
feedEntry.setActivitySummaryFormat("json");
feedEntry.setSiteNetwork("testSite2");
feedEntry.setActivityType("testActivityType");
feedEntry.setPostUserId("testUserA");
feedEntry.setFeedUserId("testUserB");
feedEntry.setFeedDate(new Date());
feedDAO.insertFeedEntry(feedEntry);
feedEntry = new ActivityFeedEntity();
feedEntry.setPostDate(new Date()); // now
feedEntry.setActivitySummaryFormat("json");
feedEntry.setSiteNetwork("testSite3");
feedEntry.setActivityType("testActivityType");
feedEntry.setPostUserId("testUserA");
feedEntry.setFeedUserId("testUserB");
feedEntry.setFeedDate(new Date());
feedDAO.insertFeedEntry(feedEntry);
assertEquals(2, activityService.getSiteFeedEntries("testSite1", "json").size());
assertEquals(2, activityService.getUserFeedEntries("testUserB", "json", null).size());
// fire the cleaner
cleaner.setMaxAgeMins(10);
cleaner.execute();
assertEquals(1, activityService.getSiteFeedEntries("testSite1", "json").size());
assertEquals(1, activityService.getUserFeedEntries("testUserB", "json", null).size());
}
public void testMaxSize() throws Exception
{
cleaner.setMaxAgeMins(0);
// insert site feed entries for "testSite4"
for (int i = 0; i < 10; i++)
{
ActivityFeedEntity feedEntry = new ActivityFeedEntity();
feedEntry.setPostDate(new Date(System.currentTimeMillis()-(i*60*1000L)));
feedEntry.setActivitySummaryFormat("json");
feedEntry.setSiteNetwork("testSite4");
feedEntry.setActivityType("testActivityType");
feedEntry.setPostUserId("testUserC");
feedEntry.setFeedUserId("");
feedEntry.setFeedDate(new Date());
feedDAO.insertFeedEntry(feedEntry);
}
// insert user feed entries for user "testUserD"
for (int i = 0; i < 10; i++)
{
ActivityFeedEntity feedEntry = new ActivityFeedEntity();
feedEntry.setPostDate(new Date(System.currentTimeMillis()-(i*60*1000L)));
feedEntry.setActivitySummaryFormat("json");
feedEntry.setSiteNetwork("testSite5");
feedEntry.setActivityType("testActivityType");
feedEntry.setPostUserId("testUserA");
feedEntry.setFeedUserId("testUserD");
feedEntry.setFeedDate(new Date());
feedDAO.insertFeedEntry(feedEntry);
}
assertEquals(10, activityService.getSiteFeedEntries("testSite4", "json").size());
assertEquals(10, activityService.getUserFeedEntries("testUserD", "json", null).size());
// fire the cleaner
cleaner.setMaxFeedSize(2);
cleaner.execute();
assertEquals(2, activityService.getSiteFeedEntries("testSite4", "json").size());
assertEquals(2, activityService.getUserFeedEntries("testUserD", "json", null).size());
Date sameTime = new Date();
// insert site feed entries for "testSite6"
for (int i = 0; i < 10; i++)
{
ActivityFeedEntity feedEntry = new ActivityFeedEntity();
feedEntry.setPostDate(sameTime);
feedEntry.setActivitySummaryFormat("json");
feedEntry.setSiteNetwork("testSite6");
feedEntry.setActivityType("testActivityType");
feedEntry.setPostUserId("testUserE");
feedEntry.setFeedUserId("");
feedEntry.setFeedDate(new Date());
feedDAO.insertFeedEntry(feedEntry);
}
// insert user feed entries for user "testUserF"
for (int i = 0; i < 10; i++)
{
ActivityFeedEntity feedEntry = new ActivityFeedEntity();
feedEntry.setPostDate(sameTime);
feedEntry.setActivitySummaryFormat("json");
feedEntry.setSiteNetwork("testSite7");
feedEntry.setActivityType("testActivityType");
feedEntry.setPostUserId("testUserA");
feedEntry.setFeedUserId("testUserF");
feedEntry.setFeedDate(new Date());
feedDAO.insertFeedEntry(feedEntry);
}
assertEquals(10, activityService.getSiteFeedEntries("testSite6", "json").size());
assertEquals(10, activityService.getUserFeedEntries("testUserF", "json", null).size());
// fire the cleaner
cleaner.setMaxFeedSize(2);
cleaner.execute();
// note: no effect, since entries at max feed size have same time (eg. to nearest minute)
assertEquals(10, activityService.getSiteFeedEntries("testSite6", "json").size());
assertEquals(10, activityService.getUserFeedEntries("testUserF", "json", null).size());
}
public void testConcurrentAccessAndRemoval() throws Exception
{
cleaner.setMaxAgeMins(1);
cleaner.setMaxFeedSize(1);
final int typeCount = 3;
int threadCount = typeCount * 10;
final CountDownLatch endLatch = new CountDownLatch(threadCount);
// Kick off the threads
for (int i = 0; i < threadCount; i++)
{
Thread thread = new Thread(""+i)
{
@Override
public void run()
{
try
{
int type = new Integer(this.getName()) % typeCount;
if (type == 0)
{
int insertCount = 10;
// insert some entries
for (int i = 0; i < insertCount; i++)
{
ActivityFeedEntity feedEntry = new ActivityFeedEntity();
feedEntry.setPostDate(new Date(System.currentTimeMillis()-(i*60*1000L)));
feedEntry.setActivitySummaryFormat("json");
feedEntry.setSiteNetwork("testSite4");
feedEntry.setActivityType("testActivityType");
feedEntry.setPostUserId("testUserC");
feedEntry.setFeedUserId("");
feedEntry.setFeedDate(new Date());
feedDAO.insertFeedEntry(feedEntry);
}
System.out.println("["+this.getName()+"] Inserted "+insertCount+" entries");
}
if (type == 1)
{
// query some entries
int selectCount = activityService.getSiteFeedEntries("testSite4", "json").size();
System.out.println("["+this.getName()+"] Selected "+selectCount+" entries");
}
if (type == 2)
{
// clean some entries
int deleteCount = cleaner.execute();
System.out.println("["+this.getName()+"] Deleted "+deleteCount+" entries");
}
}
catch (Exception e)
{
fail(e.getMessage());
}
// Notify of completion
endLatch.countDown();
}
};
thread.start();
}
// Wait for them all to be done
endLatch.await();
}
}

View File

@@ -37,6 +37,12 @@ public interface ActivityFeedDAO extends ActivitiesDAO
public int deleteFeedEntries(Date keepDate) throws SQLException;
public int deleteUserFeedEntries(String feedUserId, String format, Date keepDate) throws SQLException;
public int deleteSiteFeedEntries(String siteId, String format, Date keepDate) throws SQLException;
public List<ActivityFeedEntity> selectFeedsToClean(int maxFeedSize) throws SQLException;
public List<ActivityFeedEntity> selectUserFeedEntries(String feedUserId, String format, String siteId, boolean excludeThisUser, boolean excludeOtherUsers) throws SQLException;
public List<ActivityFeedEntity> selectSiteFeedEntries(String siteUserId, String format) throws SQLException;

View File

@@ -46,6 +46,32 @@ public class ActivityFeedDAOImpl extends IBatisSqlMapper implements ActivityFeed
return getSqlMapClient().delete("delete.activity.feed.entries.older.than.date", keepDate);
}
public int deleteSiteFeedEntries(String siteId, String format, Date keepDate) throws SQLException
{
ActivityFeedEntity params = new ActivityFeedEntity();
params.setSiteNetwork(siteId);
params.setActivitySummaryFormat(format);
params.setPostDate(keepDate);
return getSqlMapClient().delete("delete.activity.feed.for.site.entries.older.than.date", params);
}
public int deleteUserFeedEntries(String feedUserId, String format, Date keepDate) throws SQLException
{
ActivityFeedEntity params = new ActivityFeedEntity();
params.setFeedUserId(feedUserId);
params.setActivitySummaryFormat(format);
params.setPostDate(keepDate);
return getSqlMapClient().delete("delete.activity.feed.for.feeduser.entries.older.than.date", params);
}
@SuppressWarnings("unchecked")
public List<ActivityFeedEntity> selectFeedsToClean(int maxFeedSize) throws SQLException
{
return (List<ActivityFeedEntity>)getSqlMapClient().queryForList("select.activity.feed.greater.than.max", maxFeedSize);
}
@SuppressWarnings("unchecked")
public List<ActivityFeedEntity> selectUserFeedEntries(String feedUserId, String format, String siteId, boolean excludeThisUser, boolean excludeOtherUsers) throws SQLException
{