inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-07-31 17:39:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

REPO-4329 Configure Custom renditions (#554)

Browse Source 
* REPO-4329 Configure Custom renditions

- read external directories for pipelines and renditions
- bug fix: the renditions schedule was being cleared when the pipeline schedule was cleared
This commit is contained in:
alandavis
2019-08-07 21:58:59 +01:00
committed by GitHub
parent 25753797bc
commit 8c9144cb1d
6 changed files with 162 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/org/alfresco/repo/rendition2/RenditionDefinitionRegistry2Impl.java
View File

@@ -57,12 +57,18 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi
{
Map<String, RenditionDefinition2> renditionDefinitions = new HashMap();
Map<String, Set<Pair<String, Long>>> renditionsFor = new HashMap<>();
private int count = 0;
private int fileCount;
@Override
public String toString()
{
return "("+count+")";
int renditionCount = renditionDefinitions.size();
return "(renditions: "+renditionCount+" files: "+fileCount+")";
}
public void setFileCount(int fileCount)
{
this.fileCount = fileCount;
}
}
@@ -126,6 +132,10 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi
@Override
public boolean readConfig() throws IOException
{
if (configFileFinder != null)
{
configFileFinder.setFileCount(0);
}
return RenditionDefinitionRegistry2Impl.this.readConfig();
}
@@ -270,6 +280,7 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi
}
Data data = getData();
data.renditionDefinitions.put(renditionName, renditionDefinition);
data.setFileCount(configFileFinder == null ? 0 : configFileFinder.getFileCount());
}
public void unregister(String renditionName)

10
src/main/java/org/alfresco/transform/client/model/config/CombinedConfig.java
View File

@@ -74,6 +74,7 @@ public class CombinedConfig
private List<TransformNodeAndItsOrigin> allTransforms = new ArrayList<>();
private ObjectMapper jsonObjectMapper = new ObjectMapper();
private ConfigFileFinder configFileFinder;
private int tEngineCount;
static class TransformNodeAndItsOrigin
{
@@ -149,7 +150,11 @@ public class CombinedConfig
boolean successReadingConfig = true;
for (String url : urls)
{
if (!addRemoteConfig(url, remoteType))
if (addRemoteConfig(url, remoteType))
{
tEngineCount++ ;
}
else
{
successReadingConfig = false;
}
@@ -268,6 +273,9 @@ public class CombinedConfig
public void register(TransformServiceRegistryImpl registry) throws IOException
{
TransformServiceRegistryImpl.Data data = registry.getData();
data.setTEngineCount(tEngineCount);
data.setFileCount(configFileFinder.getFileCount());
List<TransformAndItsOrigin> transformers = getTransforms();
transformers.forEach(t->registry.register(t.transform, t.baseUrl, t.readFrom));
}

20
src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java
View File

@@ -31,7 +31,6 @@ import org.alfresco.util.ConfigScheduler;
import org.alfresco.util.PropertyCheck;
import org.apache.commons.logging.Log;
import org.quartz.CronExpression;
import org.quartz.SchedulerException;
import org.springframework.beans.factory.InitializingBean;
import java.io.IOException;
@@ -39,14 +38,11 @@ import java.io.Reader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.springframework.http.converter.json.Jackson2ObjectMapperFactoryBean;
import static org.alfresco.repo.rendition2.RenditionDefinition2.TIMEOUT;
@@ -61,12 +57,26 @@ public abstract class TransformServiceRegistryImpl implements TransformServiceRe
ConcurrentMap<String, ConcurrentMap<String, List<SupportedTransform>>> cachedSupportedTransformList = new ConcurrentHashMap<>();
private int transformerCount = 0;
private int transformCount = 0;
private int tEngineCount = 0;
private int fileCount;
boolean firstTime = true;
@Override
public String toString()
{
return "("+transformerCount+":"+transformCount+")";
return transformerCount == 0 && transformCount == 0 && tEngineCount == 0 && fileCount == 0
? ""
: "(transformers: "+transformerCount+" transforms: "+transformCount+" t-engines: "+tEngineCount+" files: "+fileCount+")";
}
public void setTEngineCount(int tEngineCount)
{
this.tEngineCount = tEngineCount;
}
public void setFileCount(int fileCount)
{
this.fileCount = fileCount;
}
}

137
src/main/java/org/alfresco/util/ConfigFileFinder.java
View File

@@ -31,6 +31,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.logging.Log;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
@@ -42,6 +43,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Enumeration;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
@@ -54,67 +56,54 @@ import java.util.jar.JarFile;
public abstract class ConfigFileFinder
{
private final ObjectMapper jsonObjectMapper;
private int fileCount;
public ConfigFileFinder(ObjectMapper jsonObjectMapper)
{
this.jsonObjectMapper = jsonObjectMapper;
}
public int getFileCount()
{
return fileCount;
}
public void setFileCount(int fileCount)
{
this.fileCount = fileCount;
}
public boolean readFiles(String path, Log log)
{
boolean successReadingConfig = true;
AtomicBoolean successReadingConfig = new AtomicBoolean(true);
try
{
boolean somethingRead = false;
AtomicBoolean somethingRead = new AtomicBoolean(false);
// Try reading resources in a jar
final File jarFile = new File(getClass().getProtectionDomain().getCodeSource().getLocation().getPath());
if (jarFile.isFile())
{
Enumeration<JarEntry> entries = new JarFile(jarFile).entries(); // gives ALL entries in jar
String prefix = path + "/";
List<String> names = new ArrayList<>();
while (entries.hasMoreElements())
{
final String name = entries.nextElement().getName();
if (name.startsWith(prefix) && name.length() > prefix.length())
{
names.add(name);
}
}
Collections.sort(names);
for (String name : names)
{
somethingRead = true;
successReadingConfig &= readFile(new InputStreamReader(getResourceAsStream(name)),"resource", name, null, log);
}
new JarFile(jarFile).close();
readFromJar(jarFile, path, log, successReadingConfig, somethingRead);
}
else
{
// Try reading resources from disk
URL url = getClass().getClassLoader().getResource(path);
if (url != null)
{
File root = new File(url.getPath());
String rootPath = root.getPath();
if (root.isDirectory())
{
File[] files = root.listFiles();
Arrays.sort(files, (file1, file2) -> file1.getName().compareTo(file2.getName()));
for (File file: files)
{
somethingRead = true;
successReadingConfig &= readFile(new FileReader(file), "file", file.getPath(), null, log);
}
}
else
{
somethingRead = true;
successReadingConfig = readFile(new FileReader(root), "file", rootPath, null, log);
}
String urlPath = url.getPath();
readFromDisk(urlPath, log, successReadingConfig, somethingRead);
}
}
if (!somethingRead)
if (!somethingRead.get() && new File(path).exists())
{
// Try reading files from disk
readFromDisk(path, log, successReadingConfig, somethingRead);
}
if (!somethingRead.get())
{
log.warn("No config read from "+path);
}
@@ -122,9 +111,61 @@ public abstract class ConfigFileFinder
catch (IOException e)
{
log.error("Error reading from "+path);
successReadingConfig = false;
successReadingConfig.set(false);
}
return successReadingConfig.get();
}
private void readFromJar(File jarFile, String path, Log log, AtomicBoolean successReadingConfig, AtomicBoolean somethingRead) throws IOException
{
JarFile jar = new JarFile(jarFile);
try
{
Enumeration<JarEntry> entries = jar.entries(); // gives ALL entries in jar
String prefix = path + "/";
List<String> names = new ArrayList<>();
while (entries.hasMoreElements())
{
final String name = entries.nextElement().getName();
if ((name.startsWith(prefix) && name.length() > prefix.length()) ||
(name.equals(path)))
{
names.add(name);
}
}
Collections.sort(names);
for (String name : names)
{
InputStreamReader reader = new InputStreamReader(getResourceAsStream(name));
readFromReader(successReadingConfig, somethingRead, reader, "resource", name, null, log);
}
}
finally
{
jar.close();
}
}
private void readFromDisk(String path, Log log, AtomicBoolean successReadingConfig, AtomicBoolean somethingRead) throws FileNotFoundException
{
File root = new File(path);
if (root.isDirectory())
{
File[] files = root.listFiles();
Arrays.sort(files, (file1, file2) -> file1.getName().compareTo(file2.getName()));
for (File file : files)
{
FileReader reader = new FileReader(file);
String filePath = file.getPath();
readFromReader(successReadingConfig, somethingRead, reader, "file", filePath, null, log);
}
}
else
{
FileReader reader = new FileReader(root);
String filePath = root.getPath();
readFromReader(successReadingConfig, somethingRead, reader, "file", filePath, null, log);
}
return successReadingConfig;
}
private InputStream getResourceAsStream(String resource)
@@ -133,6 +174,20 @@ public abstract class ConfigFileFinder
return in == null ? getClass().getResourceAsStream(resource) : in;
}
private void readFromReader(AtomicBoolean successReadingConfig, AtomicBoolean somethingRead,
Reader reader, String readFrom, String path, String baseUrl, Log log)
{
somethingRead.set(true);
boolean success = readFile(reader, readFrom, path, null, log);
if (success)
{
fileCount++;
}
boolean newSuccessReadingConfig = successReadingConfig.get();
newSuccessReadingConfig &= success;
successReadingConfig.set(newSuccessReadingConfig);
}
public boolean readFile(Reader reader, String readFrom, String path, String baseUrl, Log log)
{
// At the moment it is assumed the file is Json, but that does not need to be the case.

18
src/main/java/org/alfresco/util/ConfigScheduler.java
View File

@@ -36,6 +36,7 @@ import org.quartz.JobDataMap;
import org.quartz.JobDetail;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.quartz.JobKey;
import org.quartz.Scheduler;
import org.quartz.TriggerBuilder;
import org.quartz.impl.StdSchedulerFactory;
@@ -60,7 +61,7 @@ public abstract class ConfigScheduler<Data>
{
JobDataMap dataMap = context.getJobDetail().getJobDataMap();
ConfigScheduler configScheduler = (ConfigScheduler)dataMap.get(CONFIG_SCHEDULER);
boolean successReadingConfig = configScheduler.readConfigAndReplace();
boolean successReadingConfig = configScheduler.readConfigAndReplace(true);
configScheduler.changeScheduleOnStateChange(successReadingConfig);
}
}
@@ -76,6 +77,7 @@ public abstract class ConfigScheduler<Data>
private CronExpression initialAndOnErrorCronExpression;
private Scheduler scheduler;
private JobKey jobKey;
private boolean normalCronSchedule;
protected Data data;
@@ -140,7 +142,7 @@ public abstract class ConfigScheduler<Data>
}
else
{
readConfigAndReplace();
readConfigAndReplace(false);
}
}
}
@@ -155,6 +157,7 @@ public abstract class ConfigScheduler<Data>
.withIdentity(jobName)
.ofType(ConfigSchedulerJob.class)
.build();
jobKey = job.getKey();
job.getJobDataMap().put(CONFIG_SCHEDULER, this);
CronExpression cronExpression = normalCronSchedule ? this.cronExpression : initialAndOnErrorCronExpression;
CronTrigger trigger = TriggerBuilder.newTrigger()
@@ -163,6 +166,7 @@ public abstract class ConfigScheduler<Data>
.build();
scheduler.startDelayed(0);
scheduler.scheduleJob(job, trigger);
log.debug("Schedule set "+cronExpression);
}
catch (Exception e)
{
@@ -176,7 +180,9 @@ public abstract class ConfigScheduler<Data>
{
try
{
scheduler.clear();
scheduler.deleteJob(jobKey);
scheduler = null;
jobKey = null;
}
catch (Exception e)
{
@@ -185,10 +191,10 @@ public abstract class ConfigScheduler<Data>
}
}
private boolean readConfigAndReplace()
private boolean readConfigAndReplace(boolean scheduledRead)
{
boolean successReadingConfig;
log.debug("Config read started");
log.debug((scheduledRead ? "Scheduled" : "Unscheduled")+" config read started");
Data data = getData();
try
{
@@ -197,7 +203,7 @@ public abstract class ConfigScheduler<Data>
successReadingConfig = readConfig();
data = newData;
log.debug("Config read finished "+data+
(successReadingConfig ? "" : ". Config replaced but there were problems"));
(successReadingConfig ? "" : ". Config replaced but there were problems")+"\n");
}
catch (Exception e)
{

24
src/main/resources/alfresco/repository.properties
View File

@@ -1079,27 +1079,37 @@ people.search.honor.hint.useCQ=true
# Delays cron jobs after bootstrap to allow server to fully come up before jobs start
system.cronJob.startDelayMilliseconds=60000
# Schedule for reading rendition config definitions dynamically. Initially checks every 10 seconds and then switches to
# every 10 minutes after the configuration is read successfully. If there is a error later reading the config, the
# Schedule for reading mimetype config definitions dynamically. Initially checks every 10 seconds and then switches to
# every hour after the configuration is read successfully. If there is a error later reading the config, the
# checks return to every 10 seconds.
rendition.config.cronExpression=0 0/10 * * * ?
mimetype.config.cronExpression=0 30 0/1 * * ?
mimetype.config.initialAndOnError.cronExpression=0/10 * * * * ?
# Optional property to specify an external file or directory that will be read for mimetype definitions from YAML
# files (possibly added to a volume via k8 ConfigMaps).
mimetype.config.dir=shared/classes/alfresco/extension/transform/mimetypes
# Schedule for reading rendition config definitions dynamically. Initially checks every 10 seconds and then switches to
# every hour after the configuration is read successfully. If there is a error later reading the config, the
# checks return to every 10 seconds.
rendition.config.cronExpression=2 30 0/1 * * ?
rendition.config.initialAndOnError.cronExpression=0/10 * * * * ?
# Optional property to specify an external file or directory that will be read for rendition definitions from YAML
# files (possibly added to a volume via k8 ConfigMaps).
rendition.config.dir=
rendition.config.dir=shared/classes/alfresco/extension/transform/renditions
# Optional property to specify an external file or directory that will be read for transformer json config.
local.transform.pipeline.config.dir=
local.transform.pipeline.config.dir=shared/classes/alfresco/extension/transform/pipelines
# Used to disable transforms locally.
local.transform.service.enabled=true
# Schedule for reading local transform config, so that T-Engines and local pipeline config is dynamically
# picked up, or reintegrated after an outage. Initially checks every 10 seconds and then switches to every 10 minutes
# picked up, or reintegrated after an outage. Initially checks every 10 seconds and then switches to every hour
# after the configuration is read successfully. If there is a error later reading the config, the checks return to
# every 10 seconds.
local.transform.service.cronExpression=0 0/10 * * * ?
local.transform.service.cronExpression=4 30 0/1 * * ?
local.transform.service.initialAndOnError.cronExpression=0/10 * * * * ?
# Used to disable transforms that extend AbstractContentTransformer2