mirror of
https://github.com/Alfresco/alfresco-community-repo.git
synced 2025-07-31 17:39:05 +00:00
ALF-9613: numerous changes including use of ReentrantReadWriteLock for locking and introduction of a custom cleanup job that does not delete files that are in use by the cache.
git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@30066 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Matt Ward
@@ -0,0 +1,149 @@
|
||||
/*
|
||||
* Copyright (C) 2005-2011 Alfresco Software Limited.
|
||||
*
|
||||
* This file is part of Alfresco
|
||||
*
|
||||
* Alfresco is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Alfresco is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.alfresco.repo.content.caching.cleanup;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import org.alfresco.repo.content.caching.CacheFileProps;
|
||||
import org.alfresco.repo.content.caching.ContentCacheImpl;
|
||||
import org.alfresco.repo.content.caching.FileHandler;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.springframework.beans.factory.annotation.Required;
|
||||
|
||||
/**
|
||||
* Cleans up redundant cache files from the cached content file store. Once references to cache files are
|
||||
* no longer in the in-memory cache, the binary content files can be removed.
|
||||
*
|
||||
* @author Matt Ward
|
||||
*/
|
||||
public class CachedContentCleaner implements FileHandler
|
||||
{
|
||||
private static final Log log = LogFactory.getLog(CachedContentCleaner.class);
|
||||
private ContentCacheImpl cache; // impl specific functionality required
|
||||
private Integer maxDeleteWatchCount = 1;
|
||||
|
||||
public void execute()
|
||||
{
|
||||
cache.processFiles(this);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle(File cachedContentFile)
|
||||
{
|
||||
if (log.isDebugEnabled())
|
||||
{
|
||||
log.debug("handle file: " + cachedContentFile);
|
||||
}
|
||||
|
||||
CacheFileProps props = null; // don't load unless required
|
||||
String url = cache.getContentUrl(cachedContentFile);
|
||||
if (url == null)
|
||||
{
|
||||
// Not in the cache, check the properties file
|
||||
props = new CacheFileProps(cachedContentFile);
|
||||
props.load();
|
||||
url = props.getContentUrl();
|
||||
}
|
||||
|
||||
if (url != null && !cache.contains(url))
|
||||
{
|
||||
if (props == null)
|
||||
{
|
||||
props = new CacheFileProps(cachedContentFile);
|
||||
props.load();
|
||||
}
|
||||
markOrDelete(cachedContentFile, props);
|
||||
}
|
||||
else if (url == null)
|
||||
{
|
||||
// It might still be in the cache, but we were unable to determine it from the reverse lookup
|
||||
// or the properties file. Delete the file as it is most likely orphaned. If for some reason it is
|
||||
// still in the cache, cache.getReader(url) must re-cache it.
|
||||
markOrDelete(cachedContentFile, props);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Marks a file for deletion by a future run of the CachedContentCleaner. Each time a file is observed
|
||||
* by the cleaner as being ready for deletion, the deleteWatchCount is incremented until it reaches
|
||||
* maxDeleteWatchCount - in which case the next run of cleaner will really delete it.
|
||||
* <p>
|
||||
* For maxDeleteWatchCount of 1 for example, the first cleaner run will mark the file for deletion and the second
|
||||
* run will really delete it.
|
||||
* <p>
|
||||
* This offers a degree of protection over the fairly unlikely event that a reader will be obtained for a file that
|
||||
* is in the cache but gets removed from the cache and is then deleted by the cleaner before
|
||||
* the reader was consumed. A maxDeleteWatchCount of 1 should normally be fine (recommended), whilst
|
||||
* 0 would result in immediate deletion the first time the cleaner sees it as a candidate
|
||||
* for deletion (not recommended).
|
||||
*
|
||||
* @param file
|
||||
* @param props
|
||||
*/
|
||||
private void markOrDelete(File file, CacheFileProps props)
|
||||
{
|
||||
Integer deleteWatchCount = props.getDeleteWatchCount();
|
||||
|
||||
// Just in case the value has been corrupted somehow.
|
||||
if (deleteWatchCount < 0)
|
||||
deleteWatchCount = 0;
|
||||
|
||||
if (deleteWatchCount < maxDeleteWatchCount)
|
||||
{
|
||||
deleteWatchCount++;
|
||||
props.setDeleteWatchCount(deleteWatchCount);
|
||||
props.store();
|
||||
}
|
||||
else
|
||||
{
|
||||
deleteFilesNow(file);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes both the cached content file and its peer properties file that contains the
|
||||
* original content URL and deletion marker information.
|
||||
*
|
||||
* @param cacheFile Location of cached content file.
|
||||
*/
|
||||
private void deleteFilesNow(File cacheFile)
|
||||
{
|
||||
CacheFileProps props = new CacheFileProps(cacheFile);
|
||||
props.delete();
|
||||
cacheFile.delete();
|
||||
}
|
||||
|
||||
@Required
|
||||
public void setCache(ContentCacheImpl cache)
|
||||
{
|
||||
this.cache = cache;
|
||||
}
|
||||
|
||||
public void setMaxDeleteWatchCount(Integer maxDeleteWatchCount)
|
||||
{
|
||||
if (maxDeleteWatchCount < 0)
|
||||
{
|
||||
throw new IllegalArgumentException("maxDeleteWatchCount cannot be negative [value=" + maxDeleteWatchCount + "]");
|
||||
}
|
||||
this.maxDeleteWatchCount = maxDeleteWatchCount;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright (C) 2005-2011 Alfresco Software Limited.
|
||||
*
|
||||
* This file is part of Alfresco
|
||||
*
|
||||
* Alfresco is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Alfresco is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.alfresco.repo.content.caching.cleanup;
|
||||
|
||||
import org.alfresco.error.AlfrescoRuntimeException;
|
||||
import org.quartz.Job;
|
||||
import org.quartz.JobDataMap;
|
||||
import org.quartz.JobExecutionContext;
|
||||
import org.quartz.JobExecutionException;
|
||||
|
||||
/**
|
||||
* Quartz job to remove cached content files from disk once they are no longer
|
||||
* held in the in-memory cache.
|
||||
*
|
||||
* @author Matt Ward
|
||||
*/
|
||||
public class CachedContentCleanupJob implements Job
|
||||
{
|
||||
@Override
|
||||
public void execute(JobExecutionContext context) throws JobExecutionException
|
||||
{
|
||||
JobDataMap jobData = context.getJobDetail().getJobDataMap();
|
||||
CachedContentCleaner cachedContentCleaner = cachedContentCleaner(jobData);
|
||||
cachedContentCleaner.execute();
|
||||
}
|
||||
|
||||
|
||||
private CachedContentCleaner cachedContentCleaner(JobDataMap jobData)
|
||||
{
|
||||
Object cleanerObj = jobData.get("cachedContentCleaner");
|
||||
if (cleanerObj == null || !(cleanerObj instanceof CachedContentCleaner))
|
||||
{
|
||||
throw new AlfrescoRuntimeException(
|
||||
"CachedContentCleanupJob requires a valid 'cachedContentCleaner' reference");
|
||||
}
|
||||
CachedContentCleaner cleaner = (CachedContentCleaner) cleanerObj;
|
||||
return cleaner;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
/*
|
||||
* Copyright (C) 2005-2011 Alfresco Software Limited.
|
||||
*
|
||||
* This file is part of Alfresco
|
||||
*
|
||||
* Alfresco is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Alfresco is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.alfresco.repo.content.caching.cleanup;
|
||||
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.PrintWriter;
|
||||
|
||||
import org.alfresco.repo.content.caching.CacheFileProps;
|
||||
import org.alfresco.repo.content.caching.CachingContentStore;
|
||||
import org.alfresco.repo.content.caching.ContentCacheImpl;
|
||||
import org.alfresco.repo.content.caching.Key;
|
||||
import org.alfresco.service.cmr.repository.ContentReader;
|
||||
import org.alfresco.util.ApplicationContextHelper;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
|
||||
/**
|
||||
* Tests for the CachedContentCleanupJob
|
||||
*
|
||||
* @author Matt Ward
|
||||
*/
|
||||
public class CachedContentCleanupJobTest
|
||||
{
|
||||
private enum UrlSource { PROPS_FILE, REVERSE_CACHE_LOOKUP, NOT_PRESENT };
|
||||
private ApplicationContext ctx;
|
||||
private CachingContentStore cachingStore;
|
||||
private ContentCacheImpl cache;
|
||||
private File cacheRoot;
|
||||
private CachedContentCleaner cleaner;
|
||||
|
||||
@Before
|
||||
public void setUp()
|
||||
{
|
||||
String conf = "classpath:cachingstore/test-context.xml";
|
||||
String cleanerConf = "classpath:cachingstore/test-cleaner-context.xml";
|
||||
ctx = ApplicationContextHelper.getApplicationContext(new String[] { conf, cleanerConf });
|
||||
|
||||
cachingStore = (CachingContentStore) ctx.getBean("cachingContentStore");
|
||||
|
||||
cache = (ContentCacheImpl) ctx.getBean("contentCache");
|
||||
cacheRoot = cache.getCacheRoot();
|
||||
|
||||
cleaner = (CachedContentCleaner) ctx.getBean("cachedContentCleaner");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void filesNotInCacheAreDeleted()
|
||||
{
|
||||
cleaner.setMaxDeleteWatchCount(0);
|
||||
int numFiles = 300; // Must be a multiple of number of UrlSource types being tested
|
||||
File[] files = new File[300];
|
||||
for (int i = 0; i < numFiles; i++)
|
||||
{
|
||||
// Testing with a number of files. The cached file cleaner will be able to determine the 'original'
|
||||
// content URL for each file by either retrieving from the companion properties file, or performing
|
||||
// a 'reverse lookup' in the cache (i.e. cache.contains(Key.forCacheFile(...))), or there will be no
|
||||
// URL determinable for the file.
|
||||
UrlSource urlSource = UrlSource.values()[i % UrlSource.values().length];
|
||||
File cacheFile = createCacheFile(urlSource, i);
|
||||
files[i] = cacheFile;
|
||||
}
|
||||
|
||||
// Run cleaner
|
||||
cleaner.execute();
|
||||
|
||||
// check all files deleted
|
||||
for (File file : files)
|
||||
{
|
||||
assertFalse("File should have been deleted: " + file, file.exists());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void markedFilesHaveDeletionDeferredUntilCorrectPassOfCleaner()
|
||||
{
|
||||
// A non-advisable setting but useful for testing, maxDeleteWatchCount of zero
|
||||
// which should result in immediate deletion upon discovery of content no longer in the cache.
|
||||
cleaner.setMaxDeleteWatchCount(0);
|
||||
File file = createCacheFile(UrlSource.NOT_PRESENT, 0);
|
||||
|
||||
cleaner.handle(file);
|
||||
checkFilesDeleted(file);
|
||||
|
||||
// Anticipated to be the most common setting: maxDeleteWatchCount of 1.
|
||||
cleaner.setMaxDeleteWatchCount(1);
|
||||
file = createCacheFile(UrlSource.NOT_PRESENT, 0);
|
||||
|
||||
cleaner.handle(file);
|
||||
checkWatchCountForCacheFile(file, 1);
|
||||
|
||||
cleaner.handle(file);
|
||||
checkFilesDeleted(file);
|
||||
|
||||
// Check that some other arbitrary figure for maxDeleteWatchCount works correctly.
|
||||
cleaner.setMaxDeleteWatchCount(3);
|
||||
file = createCacheFile(UrlSource.NOT_PRESENT, 0);
|
||||
|
||||
cleaner.handle(file);
|
||||
checkWatchCountForCacheFile(file, 1);
|
||||
|
||||
cleaner.handle(file);
|
||||
checkWatchCountForCacheFile(file, 2);
|
||||
|
||||
cleaner.handle(file);
|
||||
checkWatchCountForCacheFile(file, 3);
|
||||
|
||||
cleaner.handle(file);
|
||||
checkFilesDeleted(file);
|
||||
}
|
||||
|
||||
|
||||
private void checkFilesDeleted(File file)
|
||||
{
|
||||
assertFalse("File should have been deleted: " + file, file.exists());
|
||||
CacheFileProps props = new CacheFileProps(file);
|
||||
assertFalse("Properties file should have been deleted, cache file: " + file, props.exists());
|
||||
}
|
||||
|
||||
|
||||
private void checkWatchCountForCacheFile(File file, Integer expectedWatchCount)
|
||||
{
|
||||
assertTrue("File should still exist: " + file, file.exists());
|
||||
CacheFileProps props = new CacheFileProps(file);
|
||||
props.load();
|
||||
assertEquals("File should contain correct deleteWatchCount", expectedWatchCount, props.getDeleteWatchCount());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void filesInCacheAreNotDeleted()
|
||||
{
|
||||
cleaner.setMaxDeleteWatchCount(0);
|
||||
|
||||
// The SlowContentStore will always give out content when asked,
|
||||
// so asking for any content will cause something to be cached.
|
||||
int numFiles = 50;
|
||||
for (int i = 0; i < numFiles; i++)
|
||||
{
|
||||
ContentReader reader = cachingStore.getReader(String.format("store://caching/store/url-%03d.bin", i));
|
||||
reader.getContentString();
|
||||
}
|
||||
|
||||
cleaner.execute();
|
||||
|
||||
for (int i = 0; i < numFiles; i++)
|
||||
{
|
||||
File cacheFile = new File(cache.getCacheFilePath(String.format("store://caching/store/url-%03d.bin", i)));
|
||||
assertTrue("File should exist", cacheFile.exists());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private File createCacheFile(UrlSource urlSource, int fileNum)
|
||||
{
|
||||
File file = new File(cacheRoot, ContentCacheImpl.createNewCacheFilePath());
|
||||
file.getParentFile().mkdirs();
|
||||
writeSampleContent(file);
|
||||
String contentUrl = String.format("protocol://some/made/up/url-%03d.bin", fileNum);
|
||||
|
||||
switch(urlSource)
|
||||
{
|
||||
case NOT_PRESENT:
|
||||
// cache won't be able to determine original content URL for the file
|
||||
break;
|
||||
case PROPS_FILE:
|
||||
// file with content URL in properties file
|
||||
CacheFileProps props = new CacheFileProps(file);
|
||||
props.setContentUrl(contentUrl);
|
||||
props.store();
|
||||
break;
|
||||
case REVERSE_CACHE_LOOKUP:
|
||||
// file with content URL in reverse lookup cache - but not 'in the cache' (forward lookup).
|
||||
cache.putIntoLookup(Key.forCacheFile(file), contentUrl);
|
||||
}
|
||||
assertTrue("File should exist", file.exists());
|
||||
return file;
|
||||
}
|
||||
|
||||
|
||||
private void writeSampleContent(File file)
|
||||
{
|
||||
try
|
||||
{
|
||||
PrintWriter writer = new PrintWriter(file);
|
||||
writer.println("Content for sample file in " + getClass().getName());
|
||||
}
|
||||
catch (Throwable e)
|
||||
{
|
||||
throw new RuntimeException("Couldn't write file: " + file, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user