inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-07-31 17:39:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

REPO-4589 Configure Mimetypes for custom renditions (#562)

Browse Source 
- ConfigFileFinder and ConfigScheduler moved to data-model for use in MimetypeMap
- Needed to force mimetypeMap to load config straight away to get tests to pass with: mimetype.config.cronExpression=0 0 0 ? JAN * 1970
This commit is contained in:
alandavis
2019-08-15 14:33:15 +01:00
committed by GitHub
parent a05e3f20ea
commit aedf05b41f
10 changed files with 22 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@@ -40,7 +40,7 @@
<dependency.alfresco-legacy-lucene.version>6.2</dependency.alfresco-legacy-lucene.version>
<dependency.alfresco-core.version>7.21</dependency.alfresco-core.version>
<dependency.alfresco-greenmail.version>6.1</dependency.alfresco-greenmail.version>
<dependency.alfresco-data-model.version>8.47</dependency.alfresco-data-model.version>
<dependency.alfresco-data-model.version>8.48</dependency.alfresco-data-model.version>
<dependency.alfresco-jlan.version>7.1</dependency.alfresco-jlan.version>
<dependency.alfresco-pdf-renderer.version>1.1</dependency.alfresco-pdf-renderer.version>
<dependency.alfresco-hb-data-sender.version>1.0.11</dependency.alfresco-hb-data-sender.version>

3
src/main/java/org/alfresco/repo/content/transform/LocalTransformServiceRegistry.java
View File

@@ -122,7 +122,6 @@ public class LocalTransformServiceRegistry extends TransformServiceRegistryImpl
public void afterPropertiesSet() throws Exception
{
PropertyCheck.mandatory(this, "mimetypeService", mimetypeService);
PropertyCheck.mandatory(this, "pipelineConfigDir", pipelineConfigDir);
PropertyCheck.mandatory(this, "properties", properties);
PropertyCheck.mandatory(this, "transformerDebug", transformerDebug);
strictMimetypeExceptions = getStrictMimetypeExceptions();
@@ -145,7 +144,7 @@ public class LocalTransformServiceRegistry extends TransformServiceRegistryImpl
}
@Override
public synchronized LocalData getData()
public LocalData getData()
{
return (LocalData)super.getData();
}

6
src/main/java/org/alfresco/repo/rendition2/RenditionDefinitionRegistry2Impl.java
View File

@@ -191,9 +191,9 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi
public void afterPropertiesSet() throws Exception
{
PropertyCheck.mandatory(this, "transformServiceRegistry", transformServiceRegistry);
PropertyCheck.mandatory(this, "renditionConfigDir", renditionConfigDir);
PropertyCheck.mandatory(this, "timeoutDefault", timeoutDefault);
PropertyCheck.mandatory(this, "jsonObjectMapper", jsonObjectMapper);
// If we have a cronExpression it indicates that we will schedule reading.
if (cronExpression != null)
{
PropertyCheck.mandatory(this, "initialAndOnErrorCronExpression", initialAndOnErrorCronExpression);
@@ -239,12 +239,12 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi
return new Data();
}
public synchronized Data getData()
public Data getData()
{
return configScheduler.getData();
}
public boolean readConfig() throws IOException
public boolean readConfig()
{
boolean successReadingConfig = configFileFinder.readFiles("alfresco/renditions", log);
if (renditionConfigDir != null && !renditionConfigDir.isBlank())

3
src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java
View File

@@ -148,6 +148,7 @@ public abstract class TransformServiceRegistryImpl implements TransformServiceRe
public void afterPropertiesSet() throws Exception
{
PropertyCheck.mandatory(this, "jsonObjectMapper", jsonObjectMapper);
// If we have a cronExpression it indicates that we will schedule reading.
if (cronExpression != null)
{
PropertyCheck.mandatory(this, "initialAndOnErrorCronExpression", initialAndOnErrorCronExpression);
@@ -162,7 +163,7 @@ public abstract class TransformServiceRegistryImpl implements TransformServiceRe
return new Data();
}
public synchronized Data getData()
public Data getData()
{
return configScheduler.getData();
}

219
src/main/java/org/alfresco/util/ConfigFileFinder.java
View File

@@ -1,219 +0,0 @@
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2019 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.util;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.logging.Log;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Enumeration;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
/**
* Used to find configuration files as resources from the jar file or from some external location. The path supplied
* to {@link #readFiles(String, Log)} may be a directory name. Normally used by ConfigScheduler.
*
* @author adavis
*/
public abstract class ConfigFileFinder
{
private final ObjectMapper jsonObjectMapper;
private int fileCount;
public ConfigFileFinder(ObjectMapper jsonObjectMapper)
{
this.jsonObjectMapper = jsonObjectMapper;
}
public int getFileCount()
{
return fileCount;
}
public void setFileCount(int fileCount)
{
this.fileCount = fileCount;
}
public boolean readFiles(String path, Log log)
{
AtomicBoolean successReadingConfig = new AtomicBoolean(true);
try
{
AtomicBoolean somethingRead = new AtomicBoolean(false);
// Try reading resources in a jar
final File jarFile = new File(getClass().getProtectionDomain().getCodeSource().getLocation().getPath());
if (jarFile.isFile())
{
readFromJar(jarFile, path, log, successReadingConfig, somethingRead);
}
else
{
// Try reading resources from disk
URL url = getClass().getClassLoader().getResource(path);
if (url != null)
{
String urlPath = url.getPath();
readFromDisk(urlPath, log, successReadingConfig, somethingRead);
}
}
if (!somethingRead.get() && new File(path).exists())
{
// Try reading files from disk
readFromDisk(path, log, successReadingConfig, somethingRead);
}
if (!somethingRead.get())
{
log.warn("No config read from "+path);
}
}
catch (IOException e)
{
log.error("Error reading from "+path);
successReadingConfig.set(false);
}
return successReadingConfig.get();
}
private void readFromJar(File jarFile, String path, Log log, AtomicBoolean successReadingConfig, AtomicBoolean somethingRead) throws IOException
{
JarFile jar = new JarFile(jarFile);
try
{
Enumeration<JarEntry> entries = jar.entries(); // gives ALL entries in jar
String prefix = path + "/";
List<String> names = new ArrayList<>();
while (entries.hasMoreElements())
{
final String name = entries.nextElement().getName();
if ((name.startsWith(prefix) && name.length() > prefix.length()) ||
(name.equals(path)))
{
names.add(name);
}
}
Collections.sort(names);
for (String name : names)
{
InputStreamReader reader = new InputStreamReader(getResourceAsStream(name));
readFromReader(successReadingConfig, somethingRead, reader, "resource", name, null, log);
}
}
finally
{
jar.close();
}
}
private void readFromDisk(String path, Log log, AtomicBoolean successReadingConfig, AtomicBoolean somethingRead) throws FileNotFoundException
{
File root = new File(path);
if (root.isDirectory())
{
File[] files = root.listFiles();
Arrays.sort(files, (file1, file2) -> file1.getName().compareTo(file2.getName()));
for (File file : files)
{
FileReader reader = new FileReader(file);
String filePath = file.getPath();
readFromReader(successReadingConfig, somethingRead, reader, "file", filePath, null, log);
}
}
else
{
FileReader reader = new FileReader(root);
String filePath = root.getPath();
readFromReader(successReadingConfig, somethingRead, reader, "file", filePath, null, log);
}
}
private InputStream getResourceAsStream(String resource)
{
final InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource);
return in == null ? getClass().getResourceAsStream(resource) : in;
}
private void readFromReader(AtomicBoolean successReadingConfig, AtomicBoolean somethingRead,
Reader reader, String readFrom, String path, String baseUrl, Log log)
{
somethingRead.set(true);
boolean success = readFile(reader, readFrom, path, null, log);
if (success)
{
fileCount++;
}
boolean newSuccessReadingConfig = successReadingConfig.get();
newSuccessReadingConfig &= success;
successReadingConfig.set(newSuccessReadingConfig);
}
public boolean readFile(Reader reader, String readFrom, String path, String baseUrl, Log log)
{
// At the moment it is assumed the file is Json, but that does not need to be the case.
// We have the path including extension.
boolean successReadingConfig = true;
try
{
JsonNode jsonNode = jsonObjectMapper.readValue(reader, new TypeReference<JsonNode>() {});
String readFromMessage = readFrom + ' ' + path;
if (log.isTraceEnabled())
{
log.trace(readFromMessage + " config is: " + jsonNode);
}
else
{
log.debug(readFromMessage + " config read");
}
readJson(jsonNode, readFromMessage, baseUrl);
}
catch (Exception e)
{
log.error("Error reading "+path);
successReadingConfig = false;
}
return successReadingConfig;
}
protected abstract void readJson(JsonNode jsonNode, String readFromMessage, String baseUrl) throws IOException;
}

229
src/main/java/org/alfresco/util/ConfigScheduler.java
View File

@@ -1,229 +0,0 @@
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2019 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.util;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.quartz.CronExpression;
import org.quartz.CronScheduleBuilder;
import org.quartz.CronTrigger;
import org.quartz.Job;
import org.quartz.JobBuilder;
import org.quartz.JobDataMap;
import org.quartz.JobDetail;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.quartz.JobKey;
import org.quartz.Scheduler;
import org.quartz.TriggerBuilder;
import org.quartz.impl.StdSchedulerFactory;
import java.io.IOException;
import java.util.Date;
/**
* Used to schedule reading of config. The config is assumed to change from time to time.
* Initially or on error the reading frequency is high but slower once no problems are reported.
* If the normal cron schedule is not set or is in the past, the config is read only once when
* {@link #run(boolean, Log, CronExpression, CronExpression)} is called.
*
* @author adavis
*/
public abstract class ConfigScheduler<Data>
{
public static class ConfigSchedulerJob implements Job
{
@Override
public void execute(JobExecutionContext context) throws JobExecutionException
{
JobDataMap dataMap = context.getJobDetail().getJobDataMap();
ConfigScheduler configScheduler = (ConfigScheduler)dataMap.get(CONFIG_SCHEDULER);
boolean successReadingConfig = configScheduler.readConfigAndReplace(true);
configScheduler.changeScheduleOnStateChange(successReadingConfig);
}
}
public static final String CONFIG_SCHEDULER = "configScheduler";
private static final Log defaultLog = LogFactory.getLog(ConfigScheduler.class);
private static StdSchedulerFactory schedulerFactory = new StdSchedulerFactory();
private final String jobName;
private Log log;
private CronExpression cronExpression;
private CronExpression initialAndOnErrorCronExpression;
private Scheduler scheduler;
private JobKey jobKey;
private boolean normalCronSchedule;
protected Data data;
private ThreadLocal<Data> threadData = ThreadLocal.withInitial(() -> data);
public ConfigScheduler(Object client)
{
jobName = client.getClass().getName()+"Job";
}
public abstract boolean readConfig() throws IOException;
public abstract Data createData();
public synchronized Data getData()
{
Data data = threadData.get();
if (data == null)
{
data = createData();
setData(data);
}
return data;
}
private synchronized void setData(Data data)
{
this.data = data;
threadData.set(data);
}
private synchronized void clearData()
{
this.data = null; // as run() should only be called multiple times in testing, it is okay to discard the
// previous data, as there should be no other Threads trying to read it, unless they are
// left over from previous tests.
threadData.remove(); // we need to pick up the initial value next time (whatever the data value is at that point)
}
/**
* This method should only be called once in production on startup generally from Spring afterPropertiesSet methods.
* In testing it is allowed to call this method multiple times, but in that case it is recommended to pass in a
* null cronExpression (or a cronExpression such as a date in the past) so the scheduler is not started. If this is
* done, the config is still read, but before the method returns.
*/
public void run(boolean enabled, Log log, CronExpression cronExpression, CronExpression initialAndOnErrorCronExpression)
{
clearPreviousSchedule();
clearData();
if (enabled)
{
this.log = log == null ? ConfigScheduler.defaultLog : log;
Date now = new Date();
if (cronExpression != null &&
initialAndOnErrorCronExpression != null &&
cronExpression.getNextValidTimeAfter(now) != null &&
initialAndOnErrorCronExpression.getNextValidTimeAfter(now) != null)
{
this.cronExpression = cronExpression;
this.initialAndOnErrorCronExpression = initialAndOnErrorCronExpression;
schedule();
}
else
{
readConfigAndReplace(false);
}
}
}
private synchronized void schedule()
{
try
{
scheduler = schedulerFactory.getScheduler();
JobDetail job = JobBuilder.newJob()
.withIdentity(jobName)
.ofType(ConfigSchedulerJob.class)
.build();
jobKey = job.getKey();
job.getJobDataMap().put(CONFIG_SCHEDULER, this);
CronExpression cronExpression = normalCronSchedule ? this.cronExpression : initialAndOnErrorCronExpression;
CronTrigger trigger = TriggerBuilder.newTrigger()
.withIdentity(jobName+"Trigger", Scheduler.DEFAULT_GROUP)
.withSchedule(CronScheduleBuilder.cronSchedule(cronExpression))
.build();
scheduler.startDelayed(0);
scheduler.scheduleJob(job, trigger);
log.debug("Schedule set "+cronExpression);
}
catch (Exception e)
{
log.error("Error scheduling "+e.getMessage());
}
}
private void clearPreviousSchedule()
{
if (scheduler != null)
{
try
{
scheduler.deleteJob(jobKey);
scheduler = null;
jobKey = null;
}
catch (Exception e)
{
log.error("Error clearing previous schedule " + e.getMessage());
}
}
}
private boolean readConfigAndReplace(boolean scheduledRead)
{
boolean successReadingConfig;
log.debug((scheduledRead ? "Scheduled" : "Unscheduled")+" config read started");
Data data = getData();
try
{
Data newData = createData();
threadData.set(newData);
successReadingConfig = readConfig();
data = newData;
log.debug("Config read finished "+data+
(successReadingConfig ? "" : ". Config replaced but there were problems")+"\n");
}
catch (Exception e)
{
successReadingConfig = false;
log.error("Config read failed. "+e.getMessage(), e);
}
setData(data);
return successReadingConfig;
}
private void changeScheduleOnStateChange(boolean successReadingConfig)
{
// Switch schedule sequence if we were on the normal schedule and we now have problems or if
// we are on the initial/error schedule and there were no errors.
if ( normalCronSchedule && !successReadingConfig ||
!normalCronSchedule && successReadingConfig)
{
normalCronSchedule = !normalCronSchedule;
clearPreviousSchedule();
schedule();
}
}
}

6
src/main/resources/alfresco/content-services-context.xml
View File

@@ -226,8 +226,14 @@
<property name="tikaConfig">
<ref bean="tikaConfig"/>
</property>
<property name="jsonObjectMapper" ref="mimetypeServiceJsonObjectMapper" />
<property name="mimetypeJsonConfigDir" value="${mimetype.config.dir}" />
<property name="cronExpression" value="${mimetype.config.cronExpression}"></property>
<property name="initialAndOnErrorCronExpression" value="${mimetype.config.initialAndOnError.cronExpression}"></property>
</bean>
<bean id="mimetypeServiceJsonObjectMapper" class="com.fasterxml.jackson.databind.ObjectMapper" />
<bean id="contentFilterLanguagesConfigService" class="org.springframework.extensions.config.xml.XMLConfigService" init-method="init">
<constructor-arg>
<bean class="org.springframework.extensions.config.source.UrlConfigSource">

2
src/main/resources/alfresco/repository.properties
View File

@@ -1087,7 +1087,7 @@ mimetype.config.initialAndOnError.cronExpression=0/10 * * * * ?
# Optional property to specify an external file or directory that will be read for mimetype definitions from YAML
# files (possibly added to a volume via k8 ConfigMaps).
mimetype.config.dir=shared/classes/alfresco/extension/transform/mimetypes
mimetype.config.dir=shared/classes/alfresco/extension/mimetypes
# Schedule for reading rendition config definitions dynamically. Initially checks every 10 seconds and then switches to
# every hour after the configuration is read successfully. If there is a error later reading the config, the

8
src/test/java/org/alfresco/transform/client/model/config/LocalTransformServiceRegistryConfigTest.java
View File

@@ -36,7 +36,6 @@ import org.junit.Test;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.quartz.CronExpression;
import org.quartz.Scheduler;
import java.io.IOException;
import java.util.ArrayList;
@@ -48,7 +47,6 @@ import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -94,7 +92,7 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg
}
@Override
public synchronized LocalData getData()
public LocalData getData()
{
return dummyData;
}
@@ -140,7 +138,7 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg
private Map<String, List<String>> officeToImageViaPdfSupportedTransformation;
private int readConfigCount;
private long startMs;
private long startMs = System.currentTimeMillis();
@Before
public void setUp() throws Exception
@@ -427,7 +425,7 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg
readConfigCount = 0;
registry.setInitialAndOnErrorCronExpression(new CronExpression(("0/2 * * ? * * *"))); // every 2 seconds rather than 10 seconds
registry.setCronExpression(new CronExpression(("0/4 * * ? * * *"))); // every 4 seconds rather than 10 mins
registry.setCronExpression(new CronExpression(("0/4 * * ? * * *"))); // every 4 seconds rather than every hour
// Sleep until a 6 second boundary, in order to make testing clearer.
// It avoids having to work out schedule offsets and extra quick runs that can otherwise take place.

3
src/test/resources/alfresco-global.properties
View File

@@ -46,3 +46,6 @@ identity-service.credentials.provider=secret
identity-service.client-socket-timeout=1000
identity-service.client-connection-timeout=3000
identity-service.authentication.enable-username-password-authentication=false
# In the past so it data is read straight away rather than being scheduled for the first time in a few milliseconds
mimetype.config.cronExpression=0 0 0 ? JAN * 1970