diff --git a/src/main/java/org/alfresco/repo/rendition2/RenditionDefinitionRegistry2Impl.java b/src/main/java/org/alfresco/repo/rendition2/RenditionDefinitionRegistry2Impl.java index c9025a33f0..193fe97526 100644 --- a/src/main/java/org/alfresco/repo/rendition2/RenditionDefinitionRegistry2Impl.java +++ b/src/main/java/org/alfresco/repo/rendition2/RenditionDefinitionRegistry2Impl.java @@ -57,12 +57,18 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi { Map renditionDefinitions = new HashMap(); Map>> renditionsFor = new HashMap<>(); - private int count = 0; + private int fileCount; @Override public String toString() { - return "("+count+")"; + int renditionCount = renditionDefinitions.size(); + return "(renditions: "+renditionCount+" files: "+fileCount+")"; + } + + public void setFileCount(int fileCount) + { + this.fileCount = fileCount; } } @@ -126,6 +132,10 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi @Override public boolean readConfig() throws IOException { + if (configFileFinder != null) + { + configFileFinder.setFileCount(0); + } return RenditionDefinitionRegistry2Impl.this.readConfig(); } @@ -270,6 +280,7 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi } Data data = getData(); data.renditionDefinitions.put(renditionName, renditionDefinition); + data.setFileCount(configFileFinder == null ? 0 : configFileFinder.getFileCount()); } public void unregister(String renditionName) diff --git a/src/main/java/org/alfresco/transform/client/model/config/CombinedConfig.java b/src/main/java/org/alfresco/transform/client/model/config/CombinedConfig.java index 7c9363b503..dbeb3f5b79 100644 --- a/src/main/java/org/alfresco/transform/client/model/config/CombinedConfig.java +++ b/src/main/java/org/alfresco/transform/client/model/config/CombinedConfig.java @@ -74,6 +74,7 @@ public class CombinedConfig private List allTransforms = new ArrayList<>(); private ObjectMapper jsonObjectMapper = new ObjectMapper(); private ConfigFileFinder configFileFinder; + private int tEngineCount; static class TransformNodeAndItsOrigin { @@ -149,7 +150,11 @@ public class CombinedConfig boolean successReadingConfig = true; for (String url : urls) { - if (!addRemoteConfig(url, remoteType)) + if (addRemoteConfig(url, remoteType)) + { + tEngineCount++ ; + } + else { successReadingConfig = false; } @@ -268,6 +273,9 @@ public class CombinedConfig public void register(TransformServiceRegistryImpl registry) throws IOException { + TransformServiceRegistryImpl.Data data = registry.getData(); + data.setTEngineCount(tEngineCount); + data.setFileCount(configFileFinder.getFileCount()); List transformers = getTransforms(); transformers.forEach(t->registry.register(t.transform, t.baseUrl, t.readFrom)); } diff --git a/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java b/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java index 51d3d78cda..0ef7565adf 100644 --- a/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java +++ b/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java @@ -31,7 +31,6 @@ import org.alfresco.util.ConfigScheduler; import org.alfresco.util.PropertyCheck; import org.apache.commons.logging.Log; import org.quartz.CronExpression; -import org.quartz.SchedulerException; import org.springframework.beans.factory.InitializingBean; import java.io.IOException; @@ -39,14 +38,11 @@ import java.io.Reader; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import org.springframework.http.converter.json.Jackson2ObjectMapperFactoryBean; import static org.alfresco.repo.rendition2.RenditionDefinition2.TIMEOUT; @@ -61,12 +57,26 @@ public abstract class TransformServiceRegistryImpl implements TransformServiceRe ConcurrentMap>> cachedSupportedTransformList = new ConcurrentHashMap<>(); private int transformerCount = 0; private int transformCount = 0; + private int tEngineCount = 0; + private int fileCount; boolean firstTime = true; @Override public String toString() { - return "("+transformerCount+":"+transformCount+")"; + return transformerCount == 0 && transformCount == 0 && tEngineCount == 0 && fileCount == 0 + ? "" + : "(transformers: "+transformerCount+" transforms: "+transformCount+" t-engines: "+tEngineCount+" files: "+fileCount+")"; + } + + public void setTEngineCount(int tEngineCount) + { + this.tEngineCount = tEngineCount; + } + + public void setFileCount(int fileCount) + { + this.fileCount = fileCount; } } diff --git a/src/main/java/org/alfresco/util/ConfigFileFinder.java b/src/main/java/org/alfresco/util/ConfigFileFinder.java index d6fd3393a3..6b796c788b 100644 --- a/src/main/java/org/alfresco/util/ConfigFileFinder.java +++ b/src/main/java/org/alfresco/util/ConfigFileFinder.java @@ -31,6 +31,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.logging.Log; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; @@ -42,6 +43,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -54,67 +56,54 @@ import java.util.jar.JarFile; public abstract class ConfigFileFinder { private final ObjectMapper jsonObjectMapper; + private int fileCount; public ConfigFileFinder(ObjectMapper jsonObjectMapper) { this.jsonObjectMapper = jsonObjectMapper; } + public int getFileCount() + { + return fileCount; + } + + public void setFileCount(int fileCount) + { + this.fileCount = fileCount; + } + public boolean readFiles(String path, Log log) { - boolean successReadingConfig = true; + AtomicBoolean successReadingConfig = new AtomicBoolean(true); try { - boolean somethingRead = false; + AtomicBoolean somethingRead = new AtomicBoolean(false); + + // Try reading resources in a jar final File jarFile = new File(getClass().getProtectionDomain().getCodeSource().getLocation().getPath()); if (jarFile.isFile()) { - Enumeration entries = new JarFile(jarFile).entries(); // gives ALL entries in jar - String prefix = path + "/"; - List names = new ArrayList<>(); - while (entries.hasMoreElements()) - { - final String name = entries.nextElement().getName(); - if (name.startsWith(prefix) && name.length() > prefix.length()) - { - names.add(name); - } - } - Collections.sort(names); - for (String name : names) - { - somethingRead = true; - successReadingConfig &= readFile(new InputStreamReader(getResourceAsStream(name)),"resource", name, null, log); - } - - new JarFile(jarFile).close(); + readFromJar(jarFile, path, log, successReadingConfig, somethingRead); } else { + // Try reading resources from disk URL url = getClass().getClassLoader().getResource(path); if (url != null) { - File root = new File(url.getPath()); - String rootPath = root.getPath(); - if (root.isDirectory()) - { - File[] files = root.listFiles(); - Arrays.sort(files, (file1, file2) -> file1.getName().compareTo(file2.getName())); - for (File file: files) - { - somethingRead = true; - successReadingConfig &= readFile(new FileReader(file), "file", file.getPath(), null, log); - } - } - else - { - somethingRead = true; - successReadingConfig = readFile(new FileReader(root), "file", rootPath, null, log); - } + String urlPath = url.getPath(); + readFromDisk(urlPath, log, successReadingConfig, somethingRead); } } - if (!somethingRead) + if (!somethingRead.get() && new File(path).exists()) + { + // Try reading files from disk + readFromDisk(path, log, successReadingConfig, somethingRead); + } + + if (!somethingRead.get()) { log.warn("No config read from "+path); } @@ -122,9 +111,61 @@ public abstract class ConfigFileFinder catch (IOException e) { log.error("Error reading from "+path); - successReadingConfig = false; + successReadingConfig.set(false); + } + return successReadingConfig.get(); + } + + private void readFromJar(File jarFile, String path, Log log, AtomicBoolean successReadingConfig, AtomicBoolean somethingRead) throws IOException + { + JarFile jar = new JarFile(jarFile); + try + { + Enumeration entries = jar.entries(); // gives ALL entries in jar + String prefix = path + "/"; + List names = new ArrayList<>(); + while (entries.hasMoreElements()) + { + final String name = entries.nextElement().getName(); + if ((name.startsWith(prefix) && name.length() > prefix.length()) || + (name.equals(path))) + { + names.add(name); + } + } + Collections.sort(names); + for (String name : names) + { + InputStreamReader reader = new InputStreamReader(getResourceAsStream(name)); + readFromReader(successReadingConfig, somethingRead, reader, "resource", name, null, log); + } + } + finally + { + jar.close(); + } + } + + private void readFromDisk(String path, Log log, AtomicBoolean successReadingConfig, AtomicBoolean somethingRead) throws FileNotFoundException + { + File root = new File(path); + if (root.isDirectory()) + { + File[] files = root.listFiles(); + Arrays.sort(files, (file1, file2) -> file1.getName().compareTo(file2.getName())); + for (File file : files) + { + FileReader reader = new FileReader(file); + String filePath = file.getPath(); + readFromReader(successReadingConfig, somethingRead, reader, "file", filePath, null, log); + } + } + else + { + FileReader reader = new FileReader(root); + String filePath = root.getPath(); + readFromReader(successReadingConfig, somethingRead, reader, "file", filePath, null, log); } - return successReadingConfig; } private InputStream getResourceAsStream(String resource) @@ -133,6 +174,20 @@ public abstract class ConfigFileFinder return in == null ? getClass().getResourceAsStream(resource) : in; } + private void readFromReader(AtomicBoolean successReadingConfig, AtomicBoolean somethingRead, + Reader reader, String readFrom, String path, String baseUrl, Log log) + { + somethingRead.set(true); + boolean success = readFile(reader, readFrom, path, null, log); + if (success) + { + fileCount++; + } + boolean newSuccessReadingConfig = successReadingConfig.get(); + newSuccessReadingConfig &= success; + successReadingConfig.set(newSuccessReadingConfig); + } + public boolean readFile(Reader reader, String readFrom, String path, String baseUrl, Log log) { // At the moment it is assumed the file is Json, but that does not need to be the case. diff --git a/src/main/java/org/alfresco/util/ConfigScheduler.java b/src/main/java/org/alfresco/util/ConfigScheduler.java index 58d54ba976..aa982b880b 100644 --- a/src/main/java/org/alfresco/util/ConfigScheduler.java +++ b/src/main/java/org/alfresco/util/ConfigScheduler.java @@ -36,6 +36,7 @@ import org.quartz.JobDataMap; import org.quartz.JobDetail; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; +import org.quartz.JobKey; import org.quartz.Scheduler; import org.quartz.TriggerBuilder; import org.quartz.impl.StdSchedulerFactory; @@ -60,7 +61,7 @@ public abstract class ConfigScheduler { JobDataMap dataMap = context.getJobDetail().getJobDataMap(); ConfigScheduler configScheduler = (ConfigScheduler)dataMap.get(CONFIG_SCHEDULER); - boolean successReadingConfig = configScheduler.readConfigAndReplace(); + boolean successReadingConfig = configScheduler.readConfigAndReplace(true); configScheduler.changeScheduleOnStateChange(successReadingConfig); } } @@ -76,6 +77,7 @@ public abstract class ConfigScheduler private CronExpression initialAndOnErrorCronExpression; private Scheduler scheduler; + private JobKey jobKey; private boolean normalCronSchedule; protected Data data; @@ -140,7 +142,7 @@ public abstract class ConfigScheduler } else { - readConfigAndReplace(); + readConfigAndReplace(false); } } } @@ -155,6 +157,7 @@ public abstract class ConfigScheduler .withIdentity(jobName) .ofType(ConfigSchedulerJob.class) .build(); + jobKey = job.getKey(); job.getJobDataMap().put(CONFIG_SCHEDULER, this); CronExpression cronExpression = normalCronSchedule ? this.cronExpression : initialAndOnErrorCronExpression; CronTrigger trigger = TriggerBuilder.newTrigger() @@ -163,6 +166,7 @@ public abstract class ConfigScheduler .build(); scheduler.startDelayed(0); scheduler.scheduleJob(job, trigger); + log.debug("Schedule set "+cronExpression); } catch (Exception e) { @@ -176,7 +180,9 @@ public abstract class ConfigScheduler { try { - scheduler.clear(); + scheduler.deleteJob(jobKey); + scheduler = null; + jobKey = null; } catch (Exception e) { @@ -185,10 +191,10 @@ public abstract class ConfigScheduler } } - private boolean readConfigAndReplace() + private boolean readConfigAndReplace(boolean scheduledRead) { boolean successReadingConfig; - log.debug("Config read started"); + log.debug((scheduledRead ? "Scheduled" : "Unscheduled")+" config read started"); Data data = getData(); try { @@ -197,7 +203,7 @@ public abstract class ConfigScheduler successReadingConfig = readConfig(); data = newData; log.debug("Config read finished "+data+ - (successReadingConfig ? "" : ". Config replaced but there were problems")); + (successReadingConfig ? "" : ". Config replaced but there were problems")+"\n"); } catch (Exception e) { diff --git a/src/main/resources/alfresco/repository.properties b/src/main/resources/alfresco/repository.properties index 9789e75dfe..74aaa37242 100644 --- a/src/main/resources/alfresco/repository.properties +++ b/src/main/resources/alfresco/repository.properties @@ -1079,27 +1079,37 @@ people.search.honor.hint.useCQ=true # Delays cron jobs after bootstrap to allow server to fully come up before jobs start system.cronJob.startDelayMilliseconds=60000 -# Schedule for reading rendition config definitions dynamically. Initially checks every 10 seconds and then switches to -# every 10 minutes after the configuration is read successfully. If there is a error later reading the config, the +# Schedule for reading mimetype config definitions dynamically. Initially checks every 10 seconds and then switches to +# every hour after the configuration is read successfully. If there is a error later reading the config, the # checks return to every 10 seconds. -rendition.config.cronExpression=0 0/10 * * * ? +mimetype.config.cronExpression=0 30 0/1 * * ? +mimetype.config.initialAndOnError.cronExpression=0/10 * * * * ? + +# Optional property to specify an external file or directory that will be read for mimetype definitions from YAML +# files (possibly added to a volume via k8 ConfigMaps). +mimetype.config.dir=shared/classes/alfresco/extension/transform/mimetypes + +# Schedule for reading rendition config definitions dynamically. Initially checks every 10 seconds and then switches to +# every hour after the configuration is read successfully. If there is a error later reading the config, the +# checks return to every 10 seconds. +rendition.config.cronExpression=2 30 0/1 * * ? rendition.config.initialAndOnError.cronExpression=0/10 * * * * ? # Optional property to specify an external file or directory that will be read for rendition definitions from YAML # files (possibly added to a volume via k8 ConfigMaps). -rendition.config.dir= +rendition.config.dir=shared/classes/alfresco/extension/transform/renditions # Optional property to specify an external file or directory that will be read for transformer json config. -local.transform.pipeline.config.dir= +local.transform.pipeline.config.dir=shared/classes/alfresco/extension/transform/pipelines # Used to disable transforms locally. local.transform.service.enabled=true # Schedule for reading local transform config, so that T-Engines and local pipeline config is dynamically -# picked up, or reintegrated after an outage. Initially checks every 10 seconds and then switches to every 10 minutes +# picked up, or reintegrated after an outage. Initially checks every 10 seconds and then switches to every hour # after the configuration is read successfully. If there is a error later reading the config, the checks return to # every 10 seconds. -local.transform.service.cronExpression=0 0/10 * * * ? +local.transform.service.cronExpression=4 30 0/1 * * ? local.transform.service.initialAndOnError.cronExpression=0/10 * * * * ? # Used to disable transforms that extend AbstractContentTransformer2