From 77820bb25b01ccca08b7a4df74b927dacef13179 Mon Sep 17 00:00:00 2001 From: Brian Long Date: Thu, 7 Jan 2021 11:41:16 -0500 Subject: [PATCH 1/4] fixed repo name --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ff5eb97..070963d 100644 --- a/pom.xml +++ b/pom.xml @@ -74,7 +74,7 @@ - inteligr8-public + inteligr8-releases Inteligr8 Releases http://repos.yateslong.us/nexus/repository/inteligr8-public default From a076eb59865044b7d91969b4dae11d8950993090 Mon Sep 17 00:00:00 2001 From: Brian Long Date: Thu, 7 Jan 2021 15:36:51 -0500 Subject: [PATCH 2/4] supporting cache the persists across runs (faster development) --- .../java/com/inteligr8/git/ExtendedGit.java | 13 +++--- .../inteligr8/git/LocalRepositoryCache.java | 43 ++++++++++++++++--- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/inteligr8/git/ExtendedGit.java b/src/main/java/com/inteligr8/git/ExtendedGit.java index 427f5fb..9e2f185 100644 --- a/src/main/java/com/inteligr8/git/ExtendedGit.java +++ b/src/main/java/com/inteligr8/git/ExtendedGit.java @@ -31,9 +31,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ExtendedGit extends CachedGit { - + + private static final Pattern gitUrlPattern = Pattern.compile("(((ssh|http(s)?)://([^@]+@)?([^:/]+)(:[0-9]+)?/)|git@([^:]+):)([\\w\\.@\\:/\\-~]+)(\\.git)"); private final Logger logger = LoggerFactory.getLogger(ExtendedGit.class); - private final Pattern gitUrlPattern = Pattern.compile("(((ssh|http(s)?)://([^@]+@)?([^:/]+)(:[0-9]+)?/)|git@([^:]+):)([\\w\\.@\\:/\\-~]+)(\\.git)"); public ExtendedGit(Git git) { super(git.getRepository()); @@ -51,10 +51,13 @@ public class ExtendedGit extends CachedGit { String gitUrl = this.getFirstRemoteUrl(); if (this.logger.isDebugEnabled()) this.logger.debug("Remote URL: " + gitUrl); - - Matcher matcher = this.gitUrlPattern.matcher(gitUrl); + return getRepositoryFullyQualifiedName(gitUrl); + } + + public static String getRepositoryFullyQualifiedName(String gitUrl) throws URISyntaxException { + Matcher matcher = gitUrlPattern.matcher(gitUrl); if (!matcher.find()) - throw new URISyntaxException(gitUrl, "The Git URL does not match the expected regular expression: " + this.gitUrlPattern.toString()); + throw new URISyntaxException(gitUrl, "The Git URL does not match the expected regular expression: " + gitUrlPattern.toString()); return matcher.group(9); } diff --git a/src/main/java/com/inteligr8/git/LocalRepositoryCache.java b/src/main/java/com/inteligr8/git/LocalRepositoryCache.java index 1b1c07d..5302d46 100644 --- a/src/main/java/com/inteligr8/git/LocalRepositoryCache.java +++ b/src/main/java/com/inteligr8/git/LocalRepositoryCache.java @@ -16,6 +16,7 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.InvalidRemoteException; import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.api.errors.TransportException; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.util.FileUtils; import org.slf4j.Logger; @@ -36,13 +37,24 @@ public class LocalRepositoryCache { private final LRUExpiringHashMap cachedGits = new LRUExpiringHashMap<>(30); private final Map gitUrlSemaphores = new HashMap<>(); final File cacheDirectory = new File(System.getProperty("java.io.tmpdir"), "git"); - private final int simultaneousProcessesPerGitRepo = 1; + private int simultaneousThreadsPerGitRepo = 1; private LocalRepositoryCache() { this.cacheDirectory.mkdir(); this.cachedGits.addListener(new RepositoryCacheMapListener()); } + /** + * This is helpful for the tweaking performance based on the environment of + * execution. A value of 1 disables concurrency which results in the + * persistence of Git Repositories between executions and disables cache + * expiration. This is great for development, testing, and command line + * usage. It is not recommended with web service frontends. + */ + public void setSimultaneousThreadsPerGitRepository(int threads) { + this.simultaneousThreadsPerGitRepo = threads; + } + @Override protected void finalize() throws Throwable { try { @@ -91,17 +103,32 @@ public class LocalRepositoryCache { synchronized (this) { semaphore = this.gitUrlSemaphores.get(url); if (semaphore == null) - this.gitUrlSemaphores.put(url, semaphore = new Semaphore(this.simultaneousProcessesPerGitRepo)); + this.gitUrlSemaphores.put(url, semaphore = new Semaphore(this.simultaneousThreadsPerGitRepo)); } + File gitRepoDirectory = null; + semaphore.acquire(); try { CachedGit git = this.cachedGits.remove(url); if (git == null) { - File gitRepoDirectory = new File(this.cacheDirectory, UUID.randomUUID().toString() + ".git"); + String directoryBaseName; + if (this.simultaneousThreadsPerGitRepo == 1) + directoryBaseName = ExtendedGit.getRepositoryFullyQualifiedName(url).replace('/', '_'); + else directoryBaseName = UUID.randomUUID().toString(); + + gitRepoDirectory = new File(this.cacheDirectory, directoryBaseName + ".git"); if (this.logger.isDebugEnabled()) this.logger.debug("Git directory cache for clone: " + gitRepoDirectory); + if (gitRepoDirectory.exists()) { + if (this.logger.isInfoEnabled()) + this.logger.info("Git Repository already exists; reusing: " + gitRepoDirectory); + git = creds != null ? new CredentialedGit(Git.open(gitRepoDirectory), creds) : new ExtendedGit(Git.open(gitRepoDirectory)); + } + } + + if (git == null) { CloneCommand clone = new CloneCommand() .setURI(url) .setDirectory(gitRepoDirectory); @@ -115,13 +142,17 @@ public class LocalRepositoryCache { this.logger.info("Cloned Git Repository: " + ((ExtendedGit)git).getRepositoryFullyQualifiedName()); } else { if (this.logger.isDebugEnabled()) - this.logger.debug("reseting Git"); + this.logger.debug("resetting Git"); git.reset().setMode(ResetType.HARD).call(); - if (branch != null) { + if (branch != null && !branch.equals(git.getRepository().getBranch())) { if (this.logger.isDebugEnabled()) this.logger.debug("switching Git branches: " + branch); - git.checkout().setName(branch).call(); + if (git.getRepository().exactRef(Constants.R_HEADS + branch) == null) { + git.checkout().setName(branch).setStartPoint("origin/" + branch).setCreateBranch(true).call(); + } else { + git.checkout().setName(branch).call(); + } } if (this.logger.isDebugEnabled()) From a58e369ada0a907cfc1c3b093b623fcb161eaf8f Mon Sep 17 00:00:00 2001 From: Brian Long Date: Mon, 11 Jan 2021 09:50:46 -0500 Subject: [PATCH 3/4] added 'GitUrl'; stopped assuming first remote --- .../java/com/inteligr8/git/CachedGit.java | 12 ++++- src/main/java/com/inteligr8/git/GitUrl.java | 44 +++++++++++++++++++ .../inteligr8/git/LocalRepositoryCache.java | 8 ++-- 3 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 src/main/java/com/inteligr8/git/GitUrl.java diff --git a/src/main/java/com/inteligr8/git/CachedGit.java b/src/main/java/com/inteligr8/git/CachedGit.java index 001b346..6083d0d 100644 --- a/src/main/java/com/inteligr8/git/CachedGit.java +++ b/src/main/java/com/inteligr8/git/CachedGit.java @@ -1,11 +1,15 @@ package com.inteligr8.git; +import java.util.List; + import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.InvalidRemoteException; import org.eclipse.jgit.api.errors.TransportException; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.URIish; public class CachedGit extends Git { @@ -21,8 +25,12 @@ public class CachedGit extends Git { super(repo); } - public String getFirstRemoteUrl() throws GitAPIException { - return this.remoteList().call().iterator().next().getURIs().iterator().next().toString(); + public URIish getRemoteUrl(String remoteName) throws GitAPIException { + List configs = this.remoteList().call(); + for (RemoteConfig config : configs) + if (config.getName().equals(remoteName)) + return config.getURIs().iterator().next(); + return null; } @Override diff --git a/src/main/java/com/inteligr8/git/GitUrl.java b/src/main/java/com/inteligr8/git/GitUrl.java new file mode 100644 index 0000000..adb1caf --- /dev/null +++ b/src/main/java/com/inteligr8/git/GitUrl.java @@ -0,0 +1,44 @@ +package com.inteligr8.git; + +import java.net.URISyntaxException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.eclipse.jgit.transport.URIish; + +public class GitUrl { + + private static final Pattern gitUrlPattern = Pattern.compile("(((ssh|http(s)?)://([^@]+@)?([^:/]+)(:[0-9]+)?/)|git@([^:]+):)([\\w\\.@\\:/\\-~]+)(\\.git)"); + + private final String url; + private final String hostname; + private final String fullyQualifiedRepositoryName; + + public GitUrl(URIish url) { + this.url = url.toString(); + + Matcher matcher = gitUrlPattern.matcher(url.toString()); + if (!matcher.find()) + throw new IllegalArgumentException("The '" + url + "' URL does not match the expected pattern: " + gitUrlPattern.toString()); + + this.hostname = matcher.group(6) != null ? matcher.group(6) : matcher.group(8); + this.fullyQualifiedRepositoryName = matcher.group(9); + } + + public GitUrl(String url) throws URISyntaxException { + this(new URIish(url)); + } + + public String getUrl() { + return this.url; + } + + public String getHostname() { + return this.hostname; + } + + public String getFullyQualifiedRepositoryName() { + return this.fullyQualifiedRepositoryName; + } + +} diff --git a/src/main/java/com/inteligr8/git/LocalRepositoryCache.java b/src/main/java/com/inteligr8/git/LocalRepositoryCache.java index 5302d46..ddec759 100644 --- a/src/main/java/com/inteligr8/git/LocalRepositoryCache.java +++ b/src/main/java/com/inteligr8/git/LocalRepositoryCache.java @@ -114,7 +114,7 @@ public class LocalRepositoryCache { if (git == null) { String directoryBaseName; if (this.simultaneousThreadsPerGitRepo == 1) - directoryBaseName = ExtendedGit.getRepositoryFullyQualifiedName(url).replace('/', '_'); + directoryBaseName = new GitUrl(url).getFullyQualifiedRepositoryName().replace('/', '_'); else directoryBaseName = UUID.randomUUID().toString(); gitRepoDirectory = new File(this.cacheDirectory, directoryBaseName + ".git"); @@ -139,7 +139,7 @@ public class LocalRepositoryCache { this.logger.debug("Cloning Git repository: " + url); git = creds != null ? new CredentialedGit(clone, creds) : new ExtendedGit(clone); if (this.logger.isInfoEnabled()) - this.logger.info("Cloned Git Repository: " + ((ExtendedGit)git).getRepositoryFullyQualifiedName()); + this.logger.info("Cloned Git Repository: " + new GitUrl(git.getRemoteUrl("origin")).getFullyQualifiedRepositoryName()); } else { if (this.logger.isDebugEnabled()) this.logger.debug("resetting Git"); @@ -163,7 +163,7 @@ public class LocalRepositoryCache { return git; } catch (URISyntaxException use) { semaphore.release(); - throw new DeveloperException(this.logger, use); + throw new IllegalArgumentException("The Git Repository URL is not properly formatted", use); } catch (IOException ie) { semaphore.release(); throw new TransportException("A I/O issue occurred", ie); @@ -178,7 +178,7 @@ public class LocalRepositoryCache { this.logger.trace("release('" + git.getRepository().getIdentifier() + "')"); try { - String url = git.getFirstRemoteUrl(); + String url = git.getRemoteUrl("origin").toString(); synchronized (this) { Semaphore semaphore = this.gitUrlSemaphores.get(url); From 3536943567a050cc911ab173b3f24afb583f54d6 Mon Sep 17 00:00:00 2001 From: Brian Long Date: Mon, 11 Jan 2021 09:50:59 -0500 Subject: [PATCH 4/4] massive refactoring of extended Git --- .../java/com/inteligr8/git/ExtendedGit.java | 139 +++++++++++------- 1 file changed, 85 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/inteligr8/git/ExtendedGit.java b/src/main/java/com/inteligr8/git/ExtendedGit.java index 9e2f185..7804b8b 100644 --- a/src/main/java/com/inteligr8/git/ExtendedGit.java +++ b/src/main/java/com/inteligr8/git/ExtendedGit.java @@ -1,38 +1,35 @@ package com.inteligr8.git; import java.io.IOException; -import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.ListBranchCommand; -import org.eclipse.jgit.api.CreateBranchCommand.SetupUpstreamMode; import org.eclipse.jgit.api.ListBranchCommand.ListMode; import org.eclipse.jgit.api.ResetCommand.ResetType; -import org.eclipse.jgit.api.errors.CheckoutConflictException; import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.api.errors.InvalidRefNameException; import org.eclipse.jgit.api.errors.InvalidRemoteException; -import org.eclipse.jgit.api.errors.RefAlreadyExistsException; -import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.api.errors.TransportException; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.URIish; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ExtendedGit extends CachedGit { - private static final Pattern gitUrlPattern = Pattern.compile("(((ssh|http(s)?)://([^@]+@)?([^:/]+)(:[0-9]+)?/)|git@([^:]+):)([\\w\\.@\\:/\\-~]+)(\\.git)"); private final Logger logger = LoggerFactory.getLogger(ExtendedGit.class); public ExtendedGit(Git git) { @@ -47,67 +44,87 @@ public class ExtendedGit extends CachedGit { super(repo); } - public String getRepositoryFullyQualifiedName() throws GitAPIException, URISyntaxException { - String gitUrl = this.getFirstRemoteUrl(); - if (this.logger.isDebugEnabled()) - this.logger.debug("Remote URL: " + gitUrl); - return getRepositoryFullyQualifiedName(gitUrl); + public Collection findRemotesByHostname(String hostname) throws GitAPIException { + List matchingRemotes = new LinkedList<>(); + + List remotes = this.remoteList().call(); + for (RemoteConfig remote : remotes) { + for (URIish uri : remote.getURIs()) { + GitUrl url = new GitUrl(uri); + if (hostname.equalsIgnoreCase(url.getHostname())) { + matchingRemotes.add(remote); + break; + } + } + } + + return matchingRemotes; } - public static String getRepositoryFullyQualifiedName(String gitUrl) throws URISyntaxException { - Matcher matcher = gitUrlPattern.matcher(gitUrl); - if (!matcher.find()) - throw new URISyntaxException(gitUrl, "The Git URL does not match the expected regular expression: " + gitUrlPattern.toString()); - return matcher.group(9); + public Collection findRemoteNames(String branchName) throws IOException, GitAPIException { + List remoteNames = new LinkedList<>(); + + List remotes = this.remoteList().call(); + for (RemoteConfig remote : remotes) + if (this.getRepository().exactRef(Constants.R_REMOTES + remote.getName() + "/" + branchName) != null) + remoteNames.add(remote.getName()); + + return remoteNames; } - /** - * Creates a local branch, if one does not already exist. - * Checks out the repository, targeting the remote branch. - * Resets the local branch up to the remote branch. - */ - public Ref normalize(String localBranchName, String exactRemoteBranchName) - throws RefAlreadyExistsException, RefNotFoundException, InvalidRefNameException, CheckoutConflictException, GitAPIException, IOException { + public Collection findRemoteRefsByName(String branchName) throws IOException, GitAPIException { + List refs = new LinkedList<>(); + + List remotes = this.remoteList().call(); + for (RemoteConfig remote : remotes) { + Ref ref = this.getRepository().exactRef(Constants.R_REMOTES + remote.getName() + "/" + branchName); + if (ref != null) + refs.add(ref); + } + + return refs; + } + + public Ref resetAndCheckout(Ref ref) throws IOException, GitAPIException { + if (!ref.getName().startsWith(Constants.R_HEADS)) + throw new IllegalArgumentException("The '" + ref + "' is not a local ref"); + return this._resetAndCheckout(ref); + + } + + public Ref resetAndCheckout(String branchName) throws IOException, GitAPIException { if (this.logger.isTraceEnabled()) - this.logger.trace("normalize('" + localBranchName + "', '" + exactRemoteBranchName + "')"); + this.logger.trace("resetOrCheckout('" + branchName + "')"); - Ref localRef = this.getRepository().exactRef(Constants.R_HEADS + localBranchName); - if (localRef == null) { - if (this.logger.isDebugEnabled()) - this.logger.debug("normalize('" + localBranchName + "', '" + exactRemoteBranchName + "'): local branch does not yet exist"); - localRef = this.branchCreate() - .setUpstreamMode(SetupUpstreamMode.NOTRACK) - .setName(localBranchName) - .setStartPoint(exactRemoteBranchName) - .call(); - if (this.logger.isDebugEnabled()) - this.logger.debug("normalize('" + localBranchName + "', '" + exactRemoteBranchName + "'): created local branch: " + localRef.getName()); + Ref localRef = this.getRepository().exactRef(Constants.R_HEADS + branchName); + if (localRef == null) + throw new IllegalArgumentException("The '" + branchName + "' branch must already exist"); + return this._resetAndCheckout(localRef); + } + + private Ref _resetAndCheckout(Ref ref) throws IOException, GitAPIException { + Ref localRef = ref; + + if (!this.status().call().isClean()) { + if (this.logger.isTraceEnabled()) + this.logger.trace("resetOrCheckout('" + ref + "'): repo is dirty"); + localRef = this.reset().setMode(ResetType.HARD).call(); } - Ref checkoutRef = this.checkout() - .setName(localBranchName) - .setStartPoint(exactRemoteBranchName) - .call(); - if (this.logger.isDebugEnabled()) - this.logger.debug("normalize('" + localBranchName + "', '" + exactRemoteBranchName + "'): checked out"); - - if (!localRef.getObjectId().getName().equals(checkoutRef.getObjectId().getName())) { - this.logger.warn("A checkout did not move the local branch to the proper commit; performing reset: " + localRef.getName()); - this.reset() - .setMode(ResetType.HARD) - .setRef(exactRemoteBranchName) - .call(); - if (this.logger.isDebugEnabled()) - this.logger.debug("normalize('" + localBranchName + "', '" + exactRemoteBranchName + "'): reset"); + if (!this.getRepository().getFullBranch().equals(localRef.getName())) { + if (this.logger.isTraceEnabled()) + this.logger.trace("resetOrCheckout('" + ref + "'): repo is on a different branch"); + localRef = this.checkout().setName(localRef.getName()).call(); } - return checkoutRef; + return localRef; } /** * This method retrieves all branches, but excludes remote branches that are tracked with a local branch. * @return */ + @Deprecated public List getAllUniqueBranches() throws GitAPIException { return this.getAllUniqueBranches(null); } @@ -116,6 +133,7 @@ public class ExtendedGit extends CachedGit { * This method retrieves all branches, but excludes remote branches that are tracked with a local branch. * @return */ + @Deprecated public List getAllUniqueBranches(String containsCommitish) throws GitAPIException { if (this.logger.isTraceEnabled()) this.logger.trace("getAllUniqueBranches('" + containsCommitish + "')"); @@ -184,5 +202,18 @@ public class ExtendedGit extends CachedGit { return trimmedBranches; } + + @Deprecated + private boolean allSameCommit(Collection refs) { + if (refs.size() < 2) + return true; + + Iterator i = refs.iterator(); + ObjectId commit = i.next().getObjectId(); + while (i.hasNext()) + if (!commit.equals(i.next().getObjectId())) + return false; + return true; + } }