From 9e3ae9c8e44e93f02050292ad4a61ae2ca6bbb70 Mon Sep 17 00:00:00 2001 From: Alan Davis Date: Mon, 16 Mar 2015 13:35:57 +0000 Subject: [PATCH] Merged HEAD-BUG-FIX (5.1/Cloud) to HEAD (5.1/Cloud) 99376: BENCH-371: BM-0004: Copy Alfresco and set up build plan - Modify FileContentStore to service 'spoof://' URLs - SpoofedTextContentReader to generate consistent text for URL - Fix FileContentStoreTest so that tests actually execute - Add override for @After so that no NPEs are thrown - Add tests for the 'spoof://' protocol - Some additional checks for IllegalArgumentException on SpoofedTextContentReader construction - Force delete of 'spoof' to return false but not fail with an exception This is handled neatly by the EagerContentStoreCleaner, which accepts that the stores may protect data according to their own needs e.g. Centera - Add spoofing option to FileFolderPerformanceTester 290 files/s for 1K files vs 460 files/s for spoofed text files (no binary streaming required) - Start FileFolderLoader component that will be used by the API git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@99502 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261 --- .../model-specific-services-context.xml | 7 + pom.xml | 18 +- .../alfresco/repo/admin/RepositoryState.java | 8 + .../content/filestore/FileContentReader.java | 2 + .../content/filestore/FileContentStore.java | 64 +++- .../filestore/SpoofedTextContentReader.java | 311 ++++++++++++++++++ .../model/filefolder/FileFolderLoader.java | 84 +++++ .../AbstractReadOnlyContentStoreTest.java | 10 +- .../content/ContentFullContextTestSuite.java | 2 + .../filestore/FileContentStoreTest.java | 59 +++- .../SpoofedTextContentReaderTest.java | 196 +++++++++++ .../alfresco/repo/model/ModelTestSuite.java | 1 + .../FileFolderPerformanceTester.java | 87 +++-- source/test-resources/log4j/log4j.properties | 3 +- 14 files changed, 791 insertions(+), 61 deletions(-) create mode 100644 source/java/org/alfresco/repo/content/filestore/SpoofedTextContentReader.java create mode 100644 source/java/org/alfresco/repo/model/filefolder/FileFolderLoader.java create mode 100644 source/test-java/org/alfresco/repo/content/filestore/SpoofedTextContentReaderTest.java diff --git a/config/alfresco/model-specific-services-context.xml b/config/alfresco/model-specific-services-context.xml index 4bd9c0c32e..8f798a11e8 100644 --- a/config/alfresco/model-specific-services-context.xml +++ b/config/alfresco/model-specific-services-context.xml @@ -147,6 +147,13 @@ + + + + + + + diff --git a/pom.xml b/pom.xml index f1a63b8ec3..6d7edeea5c 100644 --- a/pom.xml +++ b/pom.xml @@ -25,12 +25,17 @@ alfresco-mbeans ${project.version} - - org.alfresco.services - alfresco-events - 1.2.4 - - + + org.alfresco + alfresco-text-gen + 1.0-SNAPSHOT + + + org.alfresco.services + alfresco-events + 1.2.4 + + com.sun.mail javax.mail 1.5.2 @@ -995,6 +1000,7 @@ **/org/alfresco/AllUnitTestsSuite.java + **/org/alfresco/repo/model/filefolder/FileFolderPerformanceTester.java diff --git a/source/java/org/alfresco/repo/admin/RepositoryState.java b/source/java/org/alfresco/repo/admin/RepositoryState.java index 6c65ad5c9f..7eec55f22a 100644 --- a/source/java/org/alfresco/repo/admin/RepositoryState.java +++ b/source/java/org/alfresco/repo/admin/RepositoryState.java @@ -21,6 +21,8 @@ package org.alfresco.repo.admin; import java.util.concurrent.locks.ReentrantReadWriteLock; /** + * A class that maintains a thread-safe ready indicator on the current bootstrap state of the repository. + * * @author Andy * */ @@ -29,6 +31,12 @@ public class RepositoryState private boolean bootstrapping; private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + /** + * Determine if the repository is ready to use. + * + * @return true if the repository bootstrap process is still going, + * or false if the repository is ready to use + */ public boolean isBootstrapping() { this.lock.readLock().lock(); diff --git a/source/java/org/alfresco/repo/content/filestore/FileContentReader.java b/source/java/org/alfresco/repo/content/filestore/FileContentReader.java index 1b5f04f354..a48c09c389 100644 --- a/source/java/org/alfresco/repo/content/filestore/FileContentReader.java +++ b/source/java/org/alfresco/repo/content/filestore/FileContentReader.java @@ -239,7 +239,9 @@ public class FileContentReader extends AbstractContentReader /** * @return Returns false as this is a reader + * @deprecated Since 5.1. This method has no value: a file reader can never write (DH: 2015/02/17) */ + @Deprecated public boolean canWrite() { return false; // we only allow reading diff --git a/source/java/org/alfresco/repo/content/filestore/FileContentStore.java b/source/java/org/alfresco/repo/content/filestore/FileContentStore.java index 0f82245879..b55dd370b8 100644 --- a/source/java/org/alfresco/repo/content/filestore/FileContentStore.java +++ b/source/java/org/alfresco/repo/content/filestore/FileContentStore.java @@ -55,7 +55,12 @@ import org.springframework.context.event.ContextRefreshedEvent; * are generated using information from the {@link ContentContext simple content context}. *

* The file names obey, as they must, the URL naming convention - * as specified in the {@link org.alfresco.repo.content.ContentStore ContentStore interface}. + * as specified in the {@link org.alfresco.repo.content.ContentStore ContentStore interface}.
+ * The protocols handled are: + *

    + *
  • {@link #STORE_PROTOCOL store}: These URLs can be generated by this implementation and are file references within the root directory.
    • + *
  • {@link #SPOOF_PROTOCOL spoof}: These URLs are never generated by the implementation but represent spoofed binary text stream data: TODO
    • + *
* * @author Derek Hulley */ @@ -64,10 +69,11 @@ public class FileContentStore implements ApplicationContextAware, ApplicationListener { /** - * store is the new prefix for file content URLs + * store is the default prefix for file content URLs * @see ContentStore#PROTOCOL_DELIMITER */ public static final String STORE_PROTOCOL = "store"; + public static final String SPOOF_PROTOCOL = "spoof"; private static final Log logger = LogFactory.getLog(FileContentStore.class); @@ -343,10 +349,20 @@ public class FileContentStore Pair urlParts = super.getContentUrlParts(contentUrl); String protocol = urlParts.getFirst(); String relativePath = urlParts.getSecond(); + return makeFile(protocol, relativePath); + } + + /** + * Make the file based on the content URL parts. + * + * @param protocol must be {@link ContentStore#PROTOCOL_DELIMITER} for this class + */ + private File makeFile(String protocol, String relativePath) + { // Check the protocol if (!protocol.equals(FileContentStore.STORE_PROTOCOL)) { - throw new UnsupportedContentUrlException(this, contentUrl); + throw new UnsupportedContentUrlException(this, protocol + PROTOCOL_DELIMITER + relativePath); } // get the file File file = new File(rootDirectory, relativePath); @@ -367,12 +383,23 @@ public class FileContentStore /** * Performs a direct check against the file for its existence. + * For {@link #SPOOF_PROTOCOL spoofed} URLs, the URL always exists. */ @Override public boolean exists(String contentUrl) { - File file = makeFile(contentUrl); - return file.exists(); + Pair urlParts = super.getContentUrlParts(contentUrl); + String protocol = urlParts.getFirst(); + String relativePath = urlParts.getSecond(); + if (protocol.equals(SPOOF_PROTOCOL)) + { + return true; + } + else + { + File file = makeFile(protocol, relativePath); + return file.exists(); + } } /** @@ -420,9 +447,18 @@ public class FileContentStore */ public ContentReader getReader(String contentUrl) { + Pair urlParts = super.getContentUrlParts(contentUrl); + String protocol = urlParts.getFirst(); + String relativePath = urlParts.getSecond(); + // Handle the spoofed URL + if (protocol.equals(SPOOF_PROTOCOL)) + { + return new SpoofedTextContentReader(contentUrl); + } + // else, it's a real file we are after try { - File file = makeFile(contentUrl); + File file = makeFile(protocol, relativePath); ContentReader reader = null; if (file.exists()) { @@ -590,11 +626,21 @@ public class FileContentStore { throw new UnsupportedOperationException("This store is currently read-only: " + this); } - // ignore files that don't exist - File file = makeFile(contentUrl); + // Dig out protocol + Pair urlParts = super.getContentUrlParts(contentUrl); + String protocol = urlParts.getFirst(); + String relativePath = urlParts.getSecond(); + if (protocol.equals(SPOOF_PROTOCOL)) + { + // This is not a failure but the content can never actually be deleted + return false; + } + // Handle regular files based on the real files + File file = makeFile(protocol, relativePath); boolean deleted = false; if (!file.exists()) { + // File does not exist deleted = true; } else @@ -618,8 +664,6 @@ public class FileContentStore return deleted; } - - /** * Creates a new content URL. This must be supported by all * stores that are compatible with Alfresco. diff --git a/source/java/org/alfresco/repo/content/filestore/SpoofedTextContentReader.java b/source/java/org/alfresco/repo/content/filestore/SpoofedTextContentReader.java new file mode 100644 index 0000000000..d1fef8929d --- /dev/null +++ b/source/java/org/alfresco/repo/content/filestore/SpoofedTextContentReader.java @@ -0,0 +1,311 @@ +/* + * Copyright (C) 2005-2010 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.repo.content.filestore; + +import java.io.File; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.alfresco.repo.content.AbstractContentReader; +import org.alfresco.repo.content.ContentStore; +import org.alfresco.repo.content.MimetypeMap; +import org.alfresco.service.cmr.repository.ContentIOException; +import org.alfresco.service.cmr.repository.ContentReader; +import org.alfresco.textgen.TextGenerator; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; + +/** + * Provides access to text data that is generated when requested. + *

+ * The URL has the format: spoofed://{locale=en_GB,seed=12345,length=1024,strings=["Alfresco", "Cloud"]} + *

+ * The lexicon for the given locale is found by taking the language part of the locale (en in en_GB) + * and finding the resource alfresco/textgen/lexicon-stem-en.txt. + * + * @see TextGenerator + * + * @author Derek Hulley + * @since 5.1 + */ +public class SpoofedTextContentReader extends AbstractContentReader +{ + public static final String LEXICON_STEM_PATH = "alfresco/textgen/lexicon-stem-@@LOCALE@@.txt"; + public static final String KEY_LOCALE = "locale"; + public static final String KEY_SEED = "seed"; + public static final String KEY_SIZE = "size"; + public static final String KEY_WORDS = "words"; + + private static Map textGeneratorsByLocale = new HashMap(); + private static ReentrantReadWriteLock textGeneratorsLock = new ReentrantReadWriteLock(); + + private static final Log logger = LogFactory.getLog(SpoofedTextContentReader.class); + + private final TextGenerator textGenerator; + private final long seed; + private final long size; + private final String[] words; + + /** + * Get a text generator for the given locale + * + * @throws RuntimeException if the locale has no lexicon exists for the locale + */ + public static TextGenerator getTextGenerator(Locale locale) + { + textGeneratorsLock.readLock().lock(); + try + { + TextGenerator tg = textGeneratorsByLocale.get(locale); + if (tg != null) + { + return tg; + } + } + finally + { + textGeneratorsLock.readLock().unlock(); + } + // Create one + textGeneratorsLock.writeLock().lock(); + try + { + // Double check + TextGenerator tg = textGeneratorsByLocale.get(locale); + if (tg != null) + { + return tg; + } + // Create it + String lang = locale.getLanguage(); + String configPath = LEXICON_STEM_PATH.replace("@@LOCALE@@", lang); + tg = new TextGenerator(configPath); + // Store it + textGeneratorsByLocale.put(locale, tg); + // Done + return tg; + } + finally + { + textGeneratorsLock.writeLock().unlock(); + } + } + + /** + * Helper to create a content URL that represents spoofed text + * + * @param locale the text local (must be supported by an appropriate lexicon config resource) + * @param seed numerical seed to ensure repeatable sequences of random text + * @param size the size (bytes) of the text to generate + * @param words additional words with decreasing frequency + * @return the content URL + * + * @throws IllegalArgumentException if the resulting URL exceeds 255 characters + */ + @SuppressWarnings("unchecked") + public static String createContentUrl(Locale locale, long seed, long size, String ... words) + { + if (locale == null || size < 0L) + { + throw new IllegalArgumentException("Locale must be supplied and size must be zero or greater."); + } + + // Make sure that there is a text generator available + SpoofedTextContentReader.getTextGenerator(locale); + + // Build map + String url = null; + try + { + JSONObject jsonObj = new JSONObject(); + jsonObj.put(KEY_LOCALE, locale.toString()); + jsonObj.put(KEY_SEED, Long.valueOf(seed).toString()); + jsonObj.put(KEY_SIZE, Long.valueOf(size).toString()); + JSONArray jsonWords = new JSONArray(); + for (String word : words) + { + if (word == null) + { + throw new IllegalArgumentException("Words to inject into the document may not be null."); + } + jsonWords.add(word); + } + jsonObj.put(KEY_WORDS, jsonWords); + + url = FileContentStore.SPOOF_PROTOCOL + "://" + jsonObj.toString(); + if (url.length() > 255) + { + throw new IllegalArgumentException("Content URLs can be up to 255 characters. Have " + url.length() + " characters: " + url); + } + return url; + } + catch (IllegalArgumentException e) + { + // Let these out as they are + throw e; + } + catch (Exception e) + { + throw new RuntimeException("Unable to create content URL using " + locale + ", " + seed + ", " + size + ", " + words, e); + } + } + + /** + * @param url a URL describing the type of text to produce (see class comments) + */ + public SpoofedTextContentReader(String url) + { + super(url); + if (url.length() > 255) + { + throw new IllegalArgumentException("A content URL is limited to 255 characters: " + url); + } + // Split out data part + int index = url.indexOf(ContentStore.PROTOCOL_DELIMITER); + if (index <= 0 || !url.startsWith(FileContentStore.SPOOF_PROTOCOL)) + { + throw new RuntimeException("URL not supported by this reader: " + url); + } + String urlData = url.substring(index + 3, url.length()); + // Parse URL + try + { + JSONParser parser = new JSONParser(); + JSONObject mappedData = (JSONObject) parser.parse(urlData); + + String jsonLocale = mappedData.containsKey(KEY_LOCALE) ? (String) mappedData.get(KEY_LOCALE) : Locale.ENGLISH.toString(); + String jsonSeed = mappedData.containsKey(KEY_SEED) ? (String) mappedData.get(KEY_SEED) : "0"; + String jsonSize = mappedData.containsKey(KEY_SIZE) ? (String) mappedData.get(KEY_SIZE) : "1024"; + JSONArray jsonWords = mappedData.containsKey(KEY_WORDS) ? (JSONArray) mappedData.get(KEY_WORDS) : new JSONArray(); + // Get the text generator + Locale locale = new Locale(jsonLocale); + seed = Long.valueOf(jsonSeed); + size = Long.valueOf(jsonSize); + words = new String[jsonWords.size()]; + for (int i = 0; i < words.length; i++) + { + words[i] = (String) jsonWords.get(i); + } + this.textGenerator = SpoofedTextContentReader.getTextGenerator(locale); + // Set the base class storage for external information + super.setLocale(locale); + super.setEncoding("UTF-8"); + super.setMimetype(MimetypeMap.MIMETYPE_TEXT_PLAIN); + } + catch (Exception e) + { + throw new RuntimeException("Unable to interpret URL: " + url, e); + } + + } + + /** + * @return true always + */ + public boolean exists() + { + return true; + } + + /** + * @return the text generator that will make the spoofed text + */ + public TextGenerator getTextGenerator() + { + return textGenerator; + } + + /** + * @return the random seed for the spoofed text + */ + public long getSeed() + { + return seed; + } + + /** + * @return the words to add to the spoofed text + */ + public String[] getWords() + { + return words; + } + + /** + * @return spoofed text size + */ + public long getSize() + { + return size; + } + + /** + * @see File#lastModified() + */ + public long getLastModified() + { + return 0L; + } + + /** + * The URL of the write is known from the start and this method contract states + * that no consideration needs to be taken w.r.t. the stream state. + */ + @Override + protected ContentReader createReader() throws ContentIOException + { + SpoofedTextContentReader reader = new SpoofedTextContentReader(getContentUrl()); + return reader; + } + + @Override + protected ReadableByteChannel getDirectReadableChannel() throws ContentIOException + { + try + { + // Interpret the URL to generate the text + InputStream textStream = textGenerator.getInputStream(seed, size, words); + ReadableByteChannel textChannel = Channels.newChannel(textStream); + // done + if (logger.isDebugEnabled()) + { + logger.debug("Opened read channel to random text for URL: " + getContentUrl()); + } + return textChannel; + } + catch (Throwable e) + { + throw new ContentIOException("Failed to read channel: " + this, e); + } + } + + @Override + protected final void setContentUrl(String contentUrl) + { + throw new UnsupportedOperationException("The URL is static and cannot be changed."); + } +} diff --git a/source/java/org/alfresco/repo/model/filefolder/FileFolderLoader.java b/source/java/org/alfresco/repo/model/filefolder/FileFolderLoader.java new file mode 100644 index 0000000000..c01428d08d --- /dev/null +++ b/source/java/org/alfresco/repo/model/filefolder/FileFolderLoader.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2005-2015 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.repo.model.filefolder; + +import org.alfresco.repo.admin.RepositoryState; +import org.alfresco.repo.model.Repository; +import org.alfresco.service.cmr.model.FileNotFoundException; +import org.alfresco.service.transaction.TransactionService; + +/** + * Class to aid in the generation of file-folder data structures for load test purposes. + *

+ * All paths referenced are in relation to the standard Alfresco "Company Home" folder, + * which acts as the root for accessing documents and folders via many APIs. + *

+ * WARNING: This class may be used but will probably NOT be considered part of the public API i.e. + * will probably change in line with Alfresco's internal requirements; nevertheless, backward + * compatibility will be maintained where practical. + * + * @author Derek Hulley + * @since 5.1 + */ +public class FileFolderLoader +{ + private final RepositoryState repoState; + private final TransactionService transactionService; + private final Repository repositoryHelper; + + /** + * @param repoState keep track of repository readiness + * @param transactionService ensure proper rollback, where required + * @param repositoryHelper access standard repository paths + */ + public FileFolderLoader(RepositoryState repoState, TransactionService transactionService, Repository repositoryHelper) + { + this.repoState = repoState; + this.transactionService = transactionService; + this.repositoryHelper = repositoryHelper; + } + + /** + * + * @param folderPath the full path to the folder + * @param fileCount the number of files to create + * @param minFileSize the smallest file size (all sizes within 1 standard deviation of the mean) + * @param maxFileSize the largest file size (all sizes within 1 standard deviation of the mean) + * @param uniqueContentCount the total number of unique files that can be generated i.e. each file will be + * one of a total number of unique files. + * @param forceBinaryStorage true to actually write the spoofed text data to the binary store + * i.e. the physical underlying storage will have a real file + * @return the number of files successfully created + * @throws FileNotFoundException if the folder path does not exist + * @throws IllegalStateException if the repository is not ready + */ + public int createFiles( + String folderPath, + int fileCount, + long minFileSize, long maxFileSize, + boolean forceBinaryStorage, + long uniqueContentCount) throws FileNotFoundException + { + if (repoState.isBootstrapping()) + { + throw new IllegalStateException("Repository is still bootstrapping."); + } + return 0; + } +} diff --git a/source/test-java/org/alfresco/repo/content/AbstractReadOnlyContentStoreTest.java b/source/test-java/org/alfresco/repo/content/AbstractReadOnlyContentStoreTest.java index 9a2e858f59..35b7baaf26 100644 --- a/source/test-java/org/alfresco/repo/content/AbstractReadOnlyContentStoreTest.java +++ b/source/test-java/org/alfresco/repo/content/AbstractReadOnlyContentStoreTest.java @@ -18,11 +18,6 @@ */ package org.alfresco.repo.content; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.ReadableByteChannel; @@ -46,6 +41,11 @@ import org.junit.Test; import org.junit.rules.TestName; import org.springframework.context.ApplicationContext; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * Abstract base class that provides a set of tests for implementations * of {@link ContentStore}. diff --git a/source/test-java/org/alfresco/repo/content/ContentFullContextTestSuite.java b/source/test-java/org/alfresco/repo/content/ContentFullContextTestSuite.java index 35f43dd31b..d4cec1fdc9 100644 --- a/source/test-java/org/alfresco/repo/content/ContentFullContextTestSuite.java +++ b/source/test-java/org/alfresco/repo/content/ContentFullContextTestSuite.java @@ -26,6 +26,7 @@ import org.alfresco.repo.content.cleanup.ContentStoreCleanerTest; import org.alfresco.repo.content.filestore.FileContentStoreTest; import org.alfresco.repo.content.filestore.NoRandomAccessFileContentStoreTest; import org.alfresco.repo.content.filestore.ReadOnlyFileContentStoreTest; +import org.alfresco.repo.content.filestore.SpoofedTextContentReaderTest; import org.alfresco.repo.content.replication.ContentStoreReplicatorTest; import org.alfresco.repo.content.replication.ReplicatingContentStoreTest; @@ -47,6 +48,7 @@ public class ContentFullContextTestSuite extends TestSuite // These tests need a full context, at least for now suite.addTestSuite(ContentStoreCleanerTest.class); //suite.addTestSuite(CharsetFinderTest.class); + suite.addTest(new JUnit4TestAdapter(SpoofedTextContentReaderTest.class)); suite.addTest(new JUnit4TestAdapter(FileContentStoreTest.class)); suite.addTest(new JUnit4TestAdapter(NoRandomAccessFileContentStoreTest.class)); suite.addTest(new JUnit4TestAdapter(ReadOnlyFileContentStoreTest.class)); diff --git a/source/test-java/org/alfresco/repo/content/filestore/FileContentStoreTest.java b/source/test-java/org/alfresco/repo/content/filestore/FileContentStoreTest.java index d0440f9ceb..e0908ab4d2 100644 --- a/source/test-java/org/alfresco/repo/content/filestore/FileContentStoreTest.java +++ b/source/test-java/org/alfresco/repo/content/filestore/FileContentStoreTest.java @@ -18,14 +18,9 @@ */ package org.alfresco.repo.content.filestore; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.File; import java.nio.ByteBuffer; +import java.util.Locale; import org.alfresco.repo.content.AbstractWritableContentStoreTest; import org.alfresco.repo.content.ContentContext; @@ -35,12 +30,21 @@ import org.alfresco.repo.content.ContentLimitProvider.SimpleFixedLimitProvider; import org.alfresco.repo.content.ContentLimitViolationException; import org.alfresco.repo.content.ContentStore; import org.alfresco.service.cmr.repository.ContentIOException; +import org.alfresco.service.cmr.repository.ContentReader; import org.alfresco.service.cmr.repository.ContentWriter; import org.alfresco.test_category.OwnJVMTestsCategory; import org.alfresco.util.TempFileProvider; +import org.junit.After; import org.junit.Before; +import org.junit.Test; import org.junit.experimental.categories.Category; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * Tests read and write functionality for the store. * @@ -64,6 +68,13 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest getName()); store.setDeleteEmptyDirs(true); + // Do not need super class's transactions + } + + @After + public void after() + { + // Do not need super class's transactions } @Override @@ -76,6 +87,7 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest * Checks that the store disallows concurrent writers to be issued to the same URL. */ @SuppressWarnings("unused") + @Test public void testConcurrentWriteDetection() throws Exception { ByteBuffer buffer = ByteBuffer.wrap("Something".getBytes()); @@ -98,6 +110,7 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest } @Override + @Test public void testRootLocation() throws Exception { ContentStore store = getStore(); @@ -111,6 +124,7 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest * Ensures that the size is something other than -1 or Long.MAX_VALUE */ @Override + @Test public void testSpaceFree() throws Exception { ContentStore store = getStore(); @@ -123,6 +137,7 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest * Ensures that the size is something other than -1 or Long.MAX_VALUE */ @Override + @Test public void testSpaceTotal() throws Exception { ContentStore store = getStore(); @@ -135,6 +150,7 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest /** * Empty parent directories should be removed when a URL is removed. */ + @Test public void testDeleteRemovesEmptyDirs() throws Exception { ContentStore store = getStore(); @@ -161,6 +177,7 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest /** * Only non-empty directories should be deleted. */ + @Test public void testDeleteLeavesNonEmptyDirs() { ContentStore store = getStore(); @@ -198,6 +215,7 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest /** * Empty parent directories are not deleted if the store is configured not to. */ + @Test public void testNoParentDirsDeleted() throws Exception { store.setDeleteEmptyDirs(false); @@ -221,6 +239,7 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest * the expected exception. * @since Thor */ + @Test public void testWriteFileWithSizeLimit() throws Exception { ContentWriter writer = getWriter(); @@ -247,9 +266,10 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest assertTrue("Stream close not detected", writer.isClosed()); } - /* + /** * Test for MNT-12301 case. */ + @Test public void testFileAccessOutsideStoreRoot() { String url = FileContentStore.STORE_PROTOCOL + ContentStore.PROTOCOL_DELIMITER + "../somefile.bin"; @@ -295,6 +315,31 @@ public class FileContentStoreTest extends AbstractWritableContentStoreTest } } + /** + * Ensure that the store is able to produce readers for spoofed text. + * + * @since 5.1 + */ + @Test + public void testSpoofedContent() throws Exception + { + String url = SpoofedTextContentReader.createContentUrl(Locale.ENGLISH, 0L, 1024L); + ContentContext ctx = new ContentContext(null, url); + try + { + store.getWriter(ctx); + fail("FileContentStore should report that all 'spoof' content exists."); + } + catch (ContentExistsException e) + { + // Expected + } + assertFalse("Deletion should be 'false'.", store.delete(url)); + assertTrue("All spoofed content already exists!", store.exists(url)); + ContentReader reader = store.getReader(url); + assertTrue(reader instanceof SpoofedTextContentReader); + assertEquals(1024L, reader.getContentString().getBytes("UTF-8").length); + } private void assertDirExists(File root, String dir) { diff --git a/source/test-java/org/alfresco/repo/content/filestore/SpoofedTextContentReaderTest.java b/source/test-java/org/alfresco/repo/content/filestore/SpoofedTextContentReaderTest.java new file mode 100644 index 0000000000..36ea308501 --- /dev/null +++ b/source/test-java/org/alfresco/repo/content/filestore/SpoofedTextContentReaderTest.java @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2005-2010 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.repo.content.filestore; + +import java.io.InputStream; +import java.util.Locale; + +import junit.framework.TestCase; + +import org.alfresco.repo.content.AbstractContentReader; +import org.alfresco.repo.content.MimetypeMap; +import org.alfresco.service.cmr.repository.ContentIOException; +import org.alfresco.service.cmr.repository.ContentReader; +import org.alfresco.test_category.OwnJVMTestsCategory; +import org.junit.Before; +import org.junit.experimental.categories.Category; +import org.springframework.util.FileCopyUtils; + +/** + * Text spoofing as a {@link ContentReader} + * + * @see SpoofedTextContentReader + * + * @author Derek Hulley + * @since 5.1 + */ +@Category(OwnJVMTestsCategory.class) +public class SpoofedTextContentReaderTest extends TestCase +{ + @Before + public void before() + { + // Nothing + } + + public void testStaticUrlHandlingErr() + { + try + { + SpoofedTextContentReader.createContentUrl(null, 12345L, 1024L); + fail(); + } + catch (IllegalArgumentException e) + { + // Expected + } + try + { + SpoofedTextContentReader.createContentUrl(Locale.ENGLISH, 12345L, -1L); + fail(); + } + catch (IllegalArgumentException e) + { + // Expected + } + try + { + SpoofedTextContentReader.createContentUrl(Locale.ENGLISH, 12345L, 1024L, (String) null); + fail(); + } + catch (IllegalArgumentException e) + { + // Expected + } + try + { + SpoofedTextContentReader.createContentUrl(Locale.FRENCH, 12345L, 1024L); + fail(); + } + catch (RuntimeException e) + { + // Expected + } + try + { + SpoofedTextContentReader.createContentUrl( + Locale.ENGLISH, 12345L, 1024L, + "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", + "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", + "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ", "1234567890ABCDEFGHIJ"); + fail(); + } + catch (IllegalArgumentException e) + { + // Expected + } + } + + public void testStaticUrlForm_01() + { + // To URL + String url = SpoofedTextContentReader.createContentUrl(Locale.ENGLISH, 12345L, 1024L, "harry"); + assertTrue(url.startsWith("spoof://{")); + assertTrue(url.contains("\"locale\":\"en\"")); + assertTrue(url.contains("\"seed\":\"12345\"")); + assertTrue(url.contains("\"size\":\"1024\"")); + assertTrue(url.contains("\"words\":[\"harry\"]")); + assertTrue(url.endsWith("}")); + // From Reader + SpoofedTextContentReader reader = new SpoofedTextContentReader(url); + assertNotNull(reader.getTextGenerator()); + assertEquals(Locale.ENGLISH, reader.getLocale()); + assertEquals(MimetypeMap.MIMETYPE_TEXT_PLAIN, reader.getMimetype()); + assertEquals("UTF-8", reader.getEncoding()); + assertEquals(12345L, reader.getSeed()); + assertEquals(1024L, reader.getSize()); + assertNotNull(reader.getWords()); + assertEquals(1, reader.getWords().length); + assertEquals("harry", reader.getWords()[0]); + } + + public void testStaticUrlForm_02() + { + // To URL + String url = SpoofedTextContentReader.createContentUrl(Locale.ENGLISH, 12345L, 1024L); + assertTrue(url.startsWith("spoof://{")); + assertTrue(url.contains("\"locale\":\"en\"")); + assertTrue(url.contains("\"seed\":\"12345\"")); + assertTrue(url.contains("\"size\":\"1024\"")); + assertTrue(url.contains("\"words\":[]")); + assertTrue(url.endsWith("}")); + // From Reader + SpoofedTextContentReader reader = new SpoofedTextContentReader(url); + assertNotNull(reader.getTextGenerator()); + assertEquals(Locale.ENGLISH, reader.getLocale()); + assertEquals(MimetypeMap.MIMETYPE_TEXT_PLAIN, reader.getMimetype()); + assertEquals("UTF-8", reader.getEncoding()); + assertEquals(12345L, reader.getSeed()); + assertEquals(1024L, reader.getSize()); + assertNotNull(reader.getWords()); + assertEquals(0, reader.getWords().length); + } + + public void testGetContentString_01() + { + // To URL + String url = SpoofedTextContentReader.createContentUrl(Locale.ENGLISH, 12345L, 56L, "harry"); + // To Reader + ContentReader reader = new SpoofedTextContentReader(url); + String readerText = reader.getContentString(); + assertEquals("harry have voice the from countered growth invited ", readerText); + // Cannot repeat + try + { + reader.getContentString(); + fail("Should not be able to reread content."); + } + catch (ContentIOException e) + { + // Expected + } + // Get a new Reader + reader = reader.getReader(); + // Get exactly the same text + assertEquals(readerText, reader.getContentString()); + } + + public void testGetContentBinary_01() throws Exception + { + // To URL + String url = SpoofedTextContentReader.createContentUrl(Locale.ENGLISH, 12345L, 56L, "harry"); + // To Reader + ContentReader reader = new SpoofedTextContentReader(url); + InputStream is = reader.getContentInputStream(); + try + { + byte[] bytes = FileCopyUtils.copyToByteArray(is); + assertEquals(56L, bytes.length); + } + finally + { + is.close(); + } + // Compare readers + ContentReader copyOne = reader.getReader(); + ContentReader copyTwo = reader.getReader(); + // Get exactly the same binaries + assertTrue(AbstractContentReader.compareContentReaders(copyOne, copyTwo)); + } +} diff --git a/source/test-java/org/alfresco/repo/model/ModelTestSuite.java b/source/test-java/org/alfresco/repo/model/ModelTestSuite.java index dce37427c6..26c31931f7 100644 --- a/source/test-java/org/alfresco/repo/model/ModelTestSuite.java +++ b/source/test-java/org/alfresco/repo/model/ModelTestSuite.java @@ -52,6 +52,7 @@ import org.junit.runners.Suite.SuiteClasses; // These need to come afterwards, as they insert extra // interceptors which would otherwise confuse things FileFolderServiceImplTest.class, + // TODO FileFolderDuplicateChildTest.class, FileFolderServicePropagationTest.class }) diff --git a/source/test-java/org/alfresco/repo/model/filefolder/FileFolderPerformanceTester.java b/source/test-java/org/alfresco/repo/model/filefolder/FileFolderPerformanceTester.java index 747df43694..6f87b52fa7 100644 --- a/source/test-java/org/alfresco/repo/model/filefolder/FileFolderPerformanceTester.java +++ b/source/test-java/org/alfresco/repo/model/filefolder/FileFolderPerformanceTester.java @@ -24,12 +24,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import junit.framework.TestCase; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.model.ContentModel; import org.alfresco.repo.content.MimetypeMap; +import org.alfresco.repo.content.filestore.SpoofedTextContentReader; import org.alfresco.repo.content.transform.AbstractContentTransformerTest; import org.alfresco.repo.dictionary.DictionaryDAO; import org.alfresco.repo.dictionary.M2Model; @@ -41,6 +43,7 @@ import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransacti import org.alfresco.service.ServiceRegistry; import org.alfresco.service.cmr.model.FileFolderService; import org.alfresco.service.cmr.model.FileInfo; +import org.alfresco.service.cmr.repository.ContentData; import org.alfresco.service.cmr.repository.ContentWriter; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.repository.NodeService; @@ -164,9 +167,10 @@ public class FileFolderPerformanceTester extends TestCase *

* Each creation (file or folder) uses the PROPAGATION REQUIRED transaction declaration. * - * @param parentNodeRef the level zero parent - * @param randomOrder true if each thread must put the children into the folders in a random order - * @return Returns the average time (ms) to create the files only + * @param parentNodeRef the level zero parent + * @param randomOrder true if each thread must put the children into the folders in a random order + * @param realFile true if a real binary must be streamed into the node + * @return Returns the average time (ms) to create the files only */ private void buildStructure( final NodeRef parentNodeRef, @@ -175,6 +179,7 @@ public class FileFolderPerformanceTester extends TestCase final int folderCount, final int batchCount, final int filesPerBatch, + final boolean realFile, final double[] dumpPoints) { RetryingTransactionCallback createFoldersCallback = new RetryingTransactionCallback() @@ -240,11 +245,25 @@ public class FileFolderPerformanceTester extends TestCase GUID.generate(), ContentModel.TYPE_CONTENT); NodeRef nodeRef = fileInfo.getNodeRef(); - // write the content - ContentWriter writer = fileFolderService.getWriter(nodeRef); - writer.setEncoding("UTF-8"); - writer.setMimetype(MimetypeMap.MIMETYPE_TEXT_PLAIN); - writer.putContent(dataFile); + if (realFile) + { + // write the content + ContentWriter writer = fileFolderService.getWriter(nodeRef); + writer.setEncoding("UTF-8"); + writer.setMimetype(MimetypeMap.MIMETYPE_TEXT_PLAIN); + writer.putContent(dataFile); + } + else + { + // Spoof some content + String contentUrl = SpoofedTextContentReader.createContentUrl( + Locale.ENGLISH, + (long) Math.random() * 1000L, + (long) Math.random() * 1024L); + SpoofedTextContentReader reader = new SpoofedTextContentReader(contentUrl); + ContentData contentData = reader.getContentData(); + nodeService.setProperty(nodeRef, ContentModel.PROP_CONTENT, contentData); + } } // done return null; @@ -264,10 +283,11 @@ public class FileFolderPerformanceTester extends TestCase if (percentComplete > 0) { - logger.debug("\n" + + System.out.println("\n" + "[" + Thread.currentThread().getName() + "] \n" + " Created " + (currentBatchCount*filesPerBatch) + " files in each of " + folderCount + - " folders (" + (randomOrder ? "shuffled" : "in order") + "): \n" + + " folders (" + (randomOrder ? "shuffled" : "in order") + ")" + + " with " + (realFile ? "real files" : "spoofed content") + " :\n" + " Progress: " + String.format("%9.2f", percentComplete) + " percent complete \n" + " Average: " + String.format("%10.2f", average) + " ms per file \n" + " Average: " + String.format("%10.2f", 1000.0/average) + " files per second"); @@ -276,7 +296,7 @@ public class FileFolderPerformanceTester extends TestCase }; // kick off the required number of threads - logger.debug("\n" + + System.out.println("\n" + "Starting " + threadCount + " threads loading " + (batchCount * filesPerBatch) + " files in each of " + folderCount + @@ -463,8 +483,8 @@ public class FileFolderPerformanceTester extends TestCase readStructure(rootFolderRef, 1, 1, new double[] {0.25, 0.50, 0.75}); } */ - // Load: 800 ordered files per folder (into 3 folders) using 4 threads - public void test_4_ordered_3_2_100() throws Exception + /** Load: 800 ordered files per folder (into 3 folders) using 4 threads using spoofed text */ + public void test_4_ordered_3_2_100_spoofed() throws Exception { buildStructure( rootFolderRef, @@ -473,6 +493,7 @@ public class FileFolderPerformanceTester extends TestCase 3, 2, 100, + false, new double[] {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90}); System.out.println("rootFolderRef: "+rootFolderRef); @@ -480,8 +501,8 @@ public class FileFolderPerformanceTester extends TestCase readStructure(rootFolderRef, 4, 5, new double[] {0.25, 0.50, 0.75}); } - // Load: 800 shuffled files per folder (into 3 folders) using 4 threads - public void test_4_shuffled_3_2_100() throws Exception + /** Load: 800 shuffled files per folder (into 3 folders) using 4 threads using spoofed text */ + public void test_4_shuffled_3_2_100_spoofed() throws Exception { buildStructure( rootFolderRef, @@ -490,6 +511,25 @@ public class FileFolderPerformanceTester extends TestCase 3, 2, 100, + false, + new double[] {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90}); + + System.out.println("rootFolderRef: "+rootFolderRef); + + readStructure(rootFolderRef, 4, 5, new double[] {0.25, 0.50, 0.75}); + } + + /** Load: 800 shuffled files per folder (into 3 folders) using 4 threads using real text */ + public void test_4_shuffled_3_2_100_real() throws Exception + { + buildStructure( + rootFolderRef, + 4, + true, + 3, + 2, + 100, + true, new double[] {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90}); System.out.println("rootFolderRef: "+rootFolderRef); @@ -497,23 +537,6 @@ public class FileFolderPerformanceTester extends TestCase readStructure(rootFolderRef, 4, 5, new double[] {0.25, 0.50, 0.75}); } -/* - // Load: 50000 ordered files per folder (into 1 folder) using 1 thread - public void test_1_ordered_1_5000_10() throws Exception - { - buildStructure( - rootFolderRef, - 1, - false, - 1, - 5000, - 10, - new double[] {0.01, 0.02, 0.03, 0.04, 0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90}); - - readStructure(rootFolderRef, 1, 1, new double[] {0.25, 0.50, 0.75}); - } -*/ - /** * Create a bunch of files and folders in a folder and then run multi-threaded directory * listings against it. diff --git a/source/test-resources/log4j/log4j.properties b/source/test-resources/log4j/log4j.properties index 610d1e9e21..943cb8f036 100644 --- a/source/test-resources/log4j/log4j.properties +++ b/source/test-resources/log4j/log4j.properties @@ -1,2 +1,3 @@ ## Test to see that Log4J additions are picked up -log4j.logger.org.alfresco.repo.admin.Log4JHierarchyInitTest=DEBUG \ No newline at end of file +log4j.logger.org.alfresco.repo.admin.Log4JHierarchyInitTest=DEBUG +log4j.logger.org.alfresco.repo.model.filefolder.FileFolderPerformanceTester=DEBUG \ No newline at end of file