diff --git a/config/alfresco/content-services-context.xml b/config/alfresco/content-services-context.xml index b619f79fc6..fefae178e6 100644 --- a/config/alfresco/content-services-context.xml +++ b/config/alfresco/content-services-context.xml @@ -60,6 +60,9 @@ + + + diff --git a/config/alfresco/repository.properties b/config/alfresco/repository.properties index 0101978710..e4965525a4 100644 --- a/config/alfresco/repository.properties +++ b/config/alfresco/repository.properties @@ -8,6 +8,7 @@ dir.root=./alf_data dir.contentstore=${dir.root}/contentstore dir.contentstore.deleted=${dir.root}/contentstore.deleted +dir.contentstore.bucketsPerMinute=0 # ContentStore subsystem: default choice filecontentstore.subsystem.name=unencryptedContentStore diff --git a/config/alfresco/subsystems/ContentStore/unencrypted/unencrypted-store-context.xml b/config/alfresco/subsystems/ContentStore/unencrypted/unencrypted-store-context.xml index c3a029528c..2d81e13d5a 100644 --- a/config/alfresco/subsystems/ContentStore/unencrypted/unencrypted-store-context.xml +++ b/config/alfresco/subsystems/ContentStore/unencrypted/unencrypted-store-context.xml @@ -6,6 +6,7 @@ + diff --git a/source/java/org/alfresco/repo/content/filestore/FileContentStore.java b/source/java/org/alfresco/repo/content/filestore/FileContentStore.java index f55fd46fc7..e5b8dc311e 100644 --- a/source/java/org/alfresco/repo/content/filestore/FileContentStore.java +++ b/source/java/org/alfresco/repo/content/filestore/FileContentStore.java @@ -42,13 +42,12 @@ import org.alfresco.repo.content.ContentStoreCreatedEvent; import org.alfresco.repo.content.EmptyContentReader; import org.alfresco.repo.content.UnsupportedContentUrlException; import org.alfresco.service.cmr.repository.ContentIOException; -import org.alfresco.service.cmr.repository.ContentReader; -import org.alfresco.service.cmr.repository.ContentWriter; -import org.alfresco.util.Deleter; -import org.alfresco.util.GUID; -import org.alfresco.util.Pair; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.logging.Log; +import org.alfresco.service.cmr.repository.ContentReader; +import org.alfresco.service.cmr.repository.ContentWriter; +import org.alfresco.util.Deleter; +import org.alfresco.util.Pair; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; @@ -91,6 +90,7 @@ public class FileContentStore private boolean readOnly; private ApplicationContext applicationContext; private boolean deleteEmptyDirs = true; + private FileContentUrlProvider fileContentUrlProvider = new TimeBasedFileContentUrlProvider(); /** * Private: for Spring-constructed instances only. @@ -207,6 +207,10 @@ public class FileContentStore { this.readOnly = readOnly; } + + public void setFileContentUrlProvider(FileContentUrlProvider fileContentUrlProvider) { + this.fileContentUrlProvider = fileContentUrlProvider; + } /** * Generates a new URL and file appropriate to it. @@ -216,7 +220,7 @@ public class FileContentStore */ /*package*/ File createNewFile() throws IOException { - String contentUrl = FileContentStore.createNewFileStoreUrl(); + String contentUrl = fileContentUrlProvider.createNewFileStoreUrl(); return createNewFile(contentUrl); } @@ -587,25 +591,7 @@ public class FileContentStore */ public static String createNewFileStoreUrl() { - Calendar calendar = new GregorianCalendar(); - int year = calendar.get(Calendar.YEAR); - int month = calendar.get(Calendar.MONTH) + 1; // 0-based - int day = calendar.get(Calendar.DAY_OF_MONTH); - int hour = calendar.get(Calendar.HOUR_OF_DAY); - int minute = calendar.get(Calendar.MINUTE); - // create the URL - StringBuilder sb = new StringBuilder(20); - sb.append(FileContentStore.STORE_PROTOCOL) - .append(ContentStore.PROTOCOL_DELIMITER) - .append(year).append('/') - .append(month).append('/') - .append(day).append('/') - .append(hour).append('/') - .append(minute).append('/') - .append(GUID.generate()).append(".bin"); - String newContentUrl = sb.toString(); - // done - return newContentUrl; + return TimeBasedFileContentUrlProvider.createNewFileStoreUrl(0); } /** diff --git a/source/java/org/alfresco/repo/content/filestore/FileContentUrlProvider.java b/source/java/org/alfresco/repo/content/filestore/FileContentUrlProvider.java new file mode 100644 index 0000000000..06d41b484e --- /dev/null +++ b/source/java/org/alfresco/repo/content/filestore/FileContentUrlProvider.java @@ -0,0 +1,43 @@ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.repo.content.filestore; + +import org.alfresco.api.AlfrescoPublicApi; + +/** + * Provider API for file content store URL implementations + * @author Andreea Dragoi + */ +@AlfrescoPublicApi +public interface FileContentUrlProvider +{ + /** + * Content URLs must consist of a prefix or protocol followed by an implementation-specific identifier + * @return file content store URL + */ + public String createNewFileStoreUrl(); +} diff --git a/source/java/org/alfresco/repo/content/filestore/TimeBasedFileContentUrlProvider.java b/source/java/org/alfresco/repo/content/filestore/TimeBasedFileContentUrlProvider.java new file mode 100644 index 0000000000..d5af2a3630 --- /dev/null +++ b/source/java/org/alfresco/repo/content/filestore/TimeBasedFileContentUrlProvider.java @@ -0,0 +1,104 @@ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.repo.content.filestore; + +import java.util.Calendar; +import java.util.GregorianCalendar; + +import org.alfresco.repo.content.ContentStore; +import org.alfresco.util.GUID; + +/** + * Default Content URL provider for file stores. + * Content URL format is store://year/month/day/hour/minute/GUID.bin, + * but can be configured to include provision for splitting data into + * buckets within minute range through bucketsPerMinute property : + * store://year/month/day/hour/minute/bucket/GUID.bin
+ *
    + *
  • store://: prefix identifying an Alfresco content stores + * regardless of the persistence mechanism.
    • + *
  • year: year
    • + *
  • month: 1-based month of the year
    • + *
  • day: 1-based day of the month
    • + *
  • hour: 0-based hour of the day
    • + *
  • minute: 0-based minute of the hour
    • + *
  • bucket: 0-based bucket depending second of minute
    • + *
  • GUID: A unique identifier
    • + *
+ *

+ * @author Andreea Dragoi + */ + +class TimeBasedFileContentUrlProvider implements FileContentUrlProvider +{ + protected int bucketsPerMinute = 0; + + public void setBucketsPerMinute(int bucketsPerMinute) + { + this.bucketsPerMinute = bucketsPerMinute; + } + + @Override + public String createNewFileStoreUrl() + { + return createNewFileStoreUrl(bucketsPerMinute); + } + + public static String createTimeBasedPath(int bucketsPerMinute){ + Calendar calendar = new GregorianCalendar(); + int year = calendar.get(Calendar.YEAR); + int month = calendar.get(Calendar.MONTH) + 1; // 0-based + int day = calendar.get(Calendar.DAY_OF_MONTH); + int hour = calendar.get(Calendar.HOUR_OF_DAY); + int minute = calendar.get(Calendar.MINUTE); + // create the URL + StringBuilder sb = new StringBuilder(20); + sb.append(year).append('/') + .append(month).append('/') + .append(day).append('/') + .append(hour).append('/') + .append(minute).append('/'); + + if (bucketsPerMinute != 0) + { + long seconds = System.currentTimeMillis() % (60 * 1000); + int actualBucket = (int) seconds / ((60 * 1000) / bucketsPerMinute); + sb.append(actualBucket).append('/'); + } + //done + return sb.toString(); + } + + public static String createNewFileStoreUrl(int minuteBucketCount) + { + StringBuilder sb = new StringBuilder(20); + sb.append(FileContentStore.STORE_PROTOCOL); + sb.append(ContentStore.PROTOCOL_DELIMITER); + sb.append(createTimeBasedPath(minuteBucketCount)); + sb.append(GUID.generate()).append(".bin"); + return sb.toString(); + } +} diff --git a/source/java/org/alfresco/repo/content/filestore/VolumeAwareContentUrlProvider.java b/source/java/org/alfresco/repo/content/filestore/VolumeAwareContentUrlProvider.java new file mode 100644 index 0000000000..d06c7743bb --- /dev/null +++ b/source/java/org/alfresco/repo/content/filestore/VolumeAwareContentUrlProvider.java @@ -0,0 +1,78 @@ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.repo.content.filestore; + +import java.util.Random; + +import org.alfresco.repo.content.ContentStore; +import org.alfresco.util.GUID; + +/** + * Content URL provider for file stores which allows routing content from a store to a selection of filesystem volumes. + * Content is randomly distributed on configured volumes. + * Content URL format is store://volume/year/month/day/hour/minute/GUID.bin, + * As {@link TimeBasedFileContentUrlProvider TimeBasedFileContentUrlProvider} can be configured to include provision for + * splitting data into buckets within minute range + * @author Andreea Dragoi + */ +class VolumeAwareContentUrlProvider extends TimeBasedFileContentUrlProvider +{ + private String[] volumes; + private Random random = new Random(); + + /** + * @param volumeNames(name of volumes separated by comma) + */ + public VolumeAwareContentUrlProvider(String volumeNames) + { + if (volumeNames == null || volumeNames.isEmpty()) + { + throw new IllegalArgumentException("Invalid volumeNames argument"); + } + this.volumes = volumeNames.split(","); + } + + @Override + public String createNewFileStoreUrl() + { + StringBuilder sb = new StringBuilder(20); + sb.append(FileContentStore.STORE_PROTOCOL) + .append(ContentStore.PROTOCOL_DELIMITER) + .append(chooseVolume()).append("/") + .append(TimeBasedFileContentUrlProvider.createTimeBasedPath(bucketsPerMinute)) + .append(GUID.generate()).append(".bin"); + String newContentUrl = sb.toString(); + return newContentUrl; + } + + private String chooseVolume() + { + int volumesNum = volumes.length; + return volumes[random.nextInt(volumesNum)]; + } + +} diff --git a/source/java/org/alfresco/repo/tenant/TenantRoutingFileContentStore.java b/source/java/org/alfresco/repo/tenant/TenantRoutingFileContentStore.java index 8729dfd856..843ba71a16 100644 --- a/source/java/org/alfresco/repo/tenant/TenantRoutingFileContentStore.java +++ b/source/java/org/alfresco/repo/tenant/TenantRoutingFileContentStore.java @@ -33,7 +33,8 @@ import java.util.Map; import org.alfresco.repo.content.ContentLimitProvider; import org.alfresco.repo.content.ContentLimitProvider.NoLimitProvider; import org.alfresco.repo.content.ContentStore; -import org.alfresco.repo.content.filestore.FileContentStore; +import org.alfresco.repo.content.filestore.FileContentStore; +import org.alfresco.repo.content.filestore.FileContentUrlProvider; import org.springframework.context.ApplicationContext; /** @@ -41,7 +42,8 @@ import org.springframework.context.ApplicationContext; */ public class TenantRoutingFileContentStore extends AbstractTenantRoutingContentStore { - private ContentLimitProvider contentLimitProvider = new NoLimitProvider(); + private ContentLimitProvider contentLimitProvider = new NoLimitProvider(); + private FileContentUrlProvider fileContentUrlProvider; /** * Sets a new {@link ContentLimitProvider} which will provide a maximum filesize for content. @@ -49,6 +51,14 @@ public class TenantRoutingFileContentStore extends AbstractTenantRoutingContentS public void setContentLimitProvider(ContentLimitProvider contentLimitProvider) { this.contentLimitProvider = contentLimitProvider; + } + + /** + * Sets a new {@link FileContentUrlProvider} which will build the content url. + */ + public void setFileContentUrlProvider(FileContentUrlProvider fileContentUrlProvider) + { + this.fileContentUrlProvider = fileContentUrlProvider; } protected ContentStore initContentStore(ApplicationContext ctx, String contentRoot) @@ -66,7 +76,11 @@ public class TenantRoutingFileContentStore extends AbstractTenantRoutingContentS { fileContentStore.setContentLimitProvider(contentLimitProvider); } - + + if(fileContentUrlProvider != null) + { + fileContentStore.setFileContentUrlProvider(fileContentUrlProvider); + } return fileContentStore; } } diff --git a/source/test-java/org/alfresco/repo/content/filestore/BucketAwareFileContentStoreTest.java b/source/test-java/org/alfresco/repo/content/filestore/BucketAwareFileContentStoreTest.java new file mode 100644 index 0000000000..18a3360333 --- /dev/null +++ b/source/test-java/org/alfresco/repo/content/filestore/BucketAwareFileContentStoreTest.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2005-2010 Alfresco Software Limited./* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.repo.content.filestore; + +import java.io.File; + +import org.alfresco.test_category.OwnJVMTestsCategory; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import static org.junit.Assert.assertTrue; + +/** + * Tests for {@link FileContentStore FileContentStore} which uses + * {@link TimeBasedFileContentUrlProvider TimeBasedFileContentUrlProvider} + * configured for provisioning splitting data into buckets within the + * minute range + * + * @author Andreea Dragoi + * + */ +@Category(OwnJVMTestsCategory.class) +public class BucketAwareFileContentStoreTest extends FileContentStoreTest +{ + private static final int BUCKETS_PER_MINUTE = 20; + private static final int ITERATIONS = 5; + + @Before + public void before() throws Exception + { + super.before(); + + TimeBasedFileContentUrlProvider fileContentUrlProvider = new TimeBasedFileContentUrlProvider(); + // configure url provider to create buckets on minute range on a + // interval of 3 seconds (60/MINUTE_BUCKET_COUNT) + fileContentUrlProvider.setBucketsPerMinute(BUCKETS_PER_MINUTE); + store.setFileContentUrlProvider(fileContentUrlProvider); + } + + @Test + public void testBucketCreation() throws Exception + { + // create several files in a interval of ~15 seconds + // depending when the test is started files can be created on same + // minute or not + File firstFile = store.createNewFile(); + for (int i = 0; i < ITERATIONS; i++) + { + store.createNewFile(); + Thread.sleep(3000); + } + File lastFile = store.createNewFile(); + + // check the minute for first and last file created + File firstFileMinute = firstFile.getParentFile().getParentFile(); + File lastFileMinute = lastFile.getParentFile().getParentFile(); + + int createdBuckets; + int firstFileMinuteBuckets = firstFileMinute.list().length; + + if (!firstFileMinute.equals(lastFileMinute)) + { + // files are created in different minutes + int lastFileMinutesBuckets = lastFileMinute.list().length; + createdBuckets = firstFileMinuteBuckets + lastFileMinutesBuckets; + } + else + { + // files are created on same minute + createdBuckets = firstFileMinuteBuckets; + } + + // Interval of 15s + time for file creation, expecting (ITERATIONS + 1) + // buckets + assertTrue("Unexpected number of buckets created", createdBuckets == ITERATIONS + 1); + } + +} diff --git a/source/test-java/org/alfresco/repo/content/filestore/FileContentStoreTest.java b/source/test-java/org/alfresco/repo/content/filestore/FileContentStoreTest.java index 4e743a9fb0..2e63f557f3 100644 --- a/source/test-java/org/alfresco/repo/content/filestore/FileContentStoreTest.java +++ b/source/test-java/org/alfresco/repo/content/filestore/FileContentStoreTest.java @@ -28,7 +28,7 @@ package org.alfresco.repo.content.filestore; import java.io.File; import java.nio.ByteBuffer; import java.util.Locale; - + import org.alfresco.repo.content.AbstractWritableContentStoreTest; import org.alfresco.repo.content.ContentContext; import org.alfresco.repo.content.ContentExistsException; @@ -45,7 +45,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; - + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -62,7 +62,7 @@ import static org.junit.Assert.fail; @Category(OwnJVMTestsCategory.class) public class FileContentStoreTest extends AbstractWritableContentStoreTest { - private FileContentStore store; + protected FileContentStore store; @Before public void before() throws Exception diff --git a/source/test-java/org/alfresco/repo/content/filestore/VolumeAwareFileContentStoreTest.java b/source/test-java/org/alfresco/repo/content/filestore/VolumeAwareFileContentStoreTest.java new file mode 100644 index 0000000000..c3677e6a38 --- /dev/null +++ b/source/test-java/org/alfresco/repo/content/filestore/VolumeAwareFileContentStoreTest.java @@ -0,0 +1,74 @@ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.repo.content.filestore; + +import java.io.File; +import java.io.IOException; + +import org.alfresco.test_category.OwnJVMTestsCategory; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import static org.junit.Assert.assertTrue; + +/** + * Tests for {@link FileContentStore} that uses {@link VolumeAwareContentUrlProvider} + * to route content from a store to a selection of filesystem volumes + * @author Andreea Dragoi + */ +@Category(OwnJVMTestsCategory.class) +public class VolumeAwareFileContentStoreTest extends FileContentStoreTest{ + + private static final String VOLUMES = "volumeA,volumeB,volumeC"; + + @Before + public void before() throws Exception + { + super.before(); + + VolumeAwareContentUrlProvider volumeAwareContentUrlProvider = new VolumeAwareContentUrlProvider(VOLUMES); + store.setFileContentUrlProvider(volumeAwareContentUrlProvider); + } + + @Test + public void testVolumeCreation() throws IOException + { + int volumesNumber = VOLUMES.split(",").length; + // create several files + for (int i = 0; i < volumesNumber * 5 ; i++) + { + store.createNewFile(); + } + File root = new File(store.getRootLocation()); + String[] folders = root.list(); + // check if root folders contains configured volumes + for (String file : folders) + { + assertTrue("Unknown volume", VOLUMES.contains(file)); + } + assertTrue("Not all configured volumes were created", folders.length == volumesNumber); + } +}