Skip to content

Commit

Permalink
[#1563, #1579] Tests: allow a fresh clone for some test cases (#1570)
Browse files Browse the repository at this point in the history 
The test repositories are cloned only once during testing, and a full
clone is performed. This causes certain test cases that require a
shallow clone to be tested incorrectly on a full clone.

Let's allow individual tests to specify if they wish to obtain a
fresh clone of a repository.
  • Loading branch information
@aidoxe-123
aidoxe-123 authored Sep 14, 2021
1 parent 387401f commit eab69a9
Show file tree
Hide file tree
Showing 21 changed files with 150 additions and 48 deletions.
78 changes: 62 additions & 16 deletions src/main/java/reposense/report/RepoCloner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ public class RepoCloner {
private static final int MAX_NO_OF_REPOS = 2;
private static final Logger logger = LogsManager.getLogger(RepoCloner.class);

private static final boolean DEFAULT_IS_FRESH_CLONE_FOR_TEST_REQUIRED = false;

private RepoConfiguration[] configs = new RepoConfiguration[MAX_NO_OF_REPOS];
private int currentIndex = 0;
private int previousIndex = 0;
Expand All @@ -65,25 +67,34 @@ public class RepoCloner {
* Does not wait for process to finish executing.
*/
public void cloneBare(RepoConfiguration config) {
cloneBare(config, DEFAULT_IS_FRESH_CLONE_FOR_TEST_REQUIRED);
}

/**
* Spawns a process to clone the bare repository specified by {@code config}.
* Does not wait for process to finish executing.
*/
public void cloneBare(RepoConfiguration config, boolean shouldFreshClone) {
configs[currentIndex] = config;

if (!config.isShallowCloningPerformed()) {
isCurrentRepoCloned = spawnCloneProcess(config);
isCurrentRepoCloned = spawnCloneProcess(config, shouldFreshClone);
} else {
boolean didShallowPartialCloneSucceed = spawnShallowPartialCloneProcess(config);
boolean didShallowPartialCloneSucceed =
spawnShallowPartialCloneProcess(config, shouldFreshClone);
String shallowPartialBareRoot = FileUtil.getShallowPartialBareRepoPath(config).toString();

if (!didShallowPartialCloneSucceed || GitRevList.checkIsEmptyRepo(shallowPartialBareRoot)) {
isCurrentRepoCloned = spawnCloneProcess(config);
isCurrentRepoCloned = spawnCloneProcess(config, shouldFreshClone);
return;
}

List<String> graftedCommits = GitRevList.getRootCommits(shallowPartialBareRoot);
List<String> graftedCommitParents = GitCatFile.getParentsOfCommits(shallowPartialBareRoot, graftedCommits);

boolean didPartialCloneSucceed = spawnPartialCloneProcess(config);
boolean didPartialCloneSucceed = spawnPartialCloneProcess(config, shouldFreshClone);
if (!didPartialCloneSucceed) {
isCurrentRepoCloned = spawnCloneProcess(config);
isCurrentRepoCloned = spawnCloneProcess(config, shouldFreshClone);
return;
}

Expand All @@ -95,9 +106,10 @@ public void cloneBare(RepoConfiguration config) {
} catch (CommitNotFoundException e) {
sinceDate = null;
}

isCurrentRepoCloned = (sinceDate != null)
? spawnShallowCloneProcess(config, sinceDate)
: spawnCloneProcess(config);
? spawnShallowCloneProcess(config, sinceDate, shouldFreshClone)
: spawnCloneProcess(config, shouldFreshClone);
}
}

Expand Down Expand Up @@ -149,7 +161,7 @@ public void cleanup() {
* Spawns a process to clone repo specified in {@code config}. Does not wait for process to finish executing.
* Should only handle a maximum of one spawned process at any time.
*/
private boolean spawnCloneProcess(RepoConfiguration config) {
private boolean spawnCloneProcess(RepoConfiguration config, boolean shouldFreshClone) {
assert(crp == null);

try {
Expand All @@ -160,8 +172,9 @@ private boolean spawnCloneProcess(RepoConfiguration config) {
Path repoDirectoryPath = getRepoParentFolder(config);
Path repoPath = Paths.get(repoDirectoryPath.toString(), config.getRepoName());

if (SystemUtil.isTestEnvironment() && Files.exists(repoPath)) {
logger.info("Skipped cloning from " + config.getLocation() + " as it was cloned before.");
if (SystemUtil.isTestEnvironment() && Files.exists(repoPath) && !shouldFreshClone) {
logger.info("Skipped cloning from " + config.getLocation()
+ " as it was cloned before and cloning is not forced.");
return true;
}

Expand All @@ -185,12 +198,24 @@ private boolean spawnCloneProcess(RepoConfiguration config) {
* Does not wait for process to finish executing.
* Should only handle a maximum of one spawned process at any time.
*/
private boolean spawnShallowCloneProcess(RepoConfiguration config, Date shallowSinceDate) {
private boolean spawnShallowCloneProcess(RepoConfiguration config, Date shallowSinceDate,
boolean shouldFreshClone) {
assert(crp == null);

try {
FileUtil.deleteDirectory(FileUtil.getBareRepoPath(config).toString());
if (!SystemUtil.isTestEnvironment()) {
FileUtil.deleteDirectory(FileUtil.getBareRepoPath(config).toString());
}

Path repoDirectoryPath = getRepoParentFolder(config);
Path repoPath = Paths.get(repoDirectoryPath.toString(), config.getRepoName());

if (SystemUtil.isTestEnvironment() && Files.exists(repoPath) && !shouldFreshClone) {
logger.info("Skipped cloning from " + config.getLocation()
+ " as it was cloned before and cloning is not forced.");
return true;
}

Files.createDirectories(repoDirectoryPath);

Path outputDirectory = Paths.get(repoDirectoryPath.toString(),
Expand All @@ -210,12 +235,23 @@ private boolean spawnShallowCloneProcess(RepoConfiguration config, Date shallowS
* Spawns a process to create partial clone of repo specified in {@code config}.
* Waits for process to finish executing.
*/
private boolean spawnPartialCloneProcess(RepoConfiguration config) {
private boolean spawnPartialCloneProcess(RepoConfiguration config, boolean shouldFreshClone) {
assert(crp == null);

try {
FileUtil.deleteDirectory(FileUtil.getPartialBareRepoPath(config).toString());
if (!SystemUtil.isTestEnvironment()) {
FileUtil.deleteDirectory(FileUtil.getPartialBareRepoPath(config).toString());
}

Path repoDirectoryPath = getRepoParentFolder(config);
Path repoPath = Paths.get(repoDirectoryPath.toString(), config.getRepoName());

if (SystemUtil.isTestEnvironment() && Files.exists(repoPath) && !shouldFreshClone) {
logger.info("Skipped cloning from " + config.getLocation()
+ " as it was cloned before and cloning is not forced.");
return true;
}

Files.createDirectories(repoDirectoryPath);

Path outputDirectory = Paths.get(repoDirectoryPath.toString(),
Expand All @@ -234,12 +270,22 @@ private boolean spawnPartialCloneProcess(RepoConfiguration config) {
* Spawns a process to create shallow partial clone of repo specified in {@code config}.
* Waits for process to finish executing.
*/
private boolean spawnShallowPartialCloneProcess(RepoConfiguration config) {
private boolean spawnShallowPartialCloneProcess(RepoConfiguration config, boolean shouldFreshClone) {
assert(crp == null);

try {
FileUtil.deleteDirectory(FileUtil.getShallowPartialBareRepoPath(config).toString());
if (!SystemUtil.isTestEnvironment()) {
FileUtil.deleteDirectory(FileUtil.getShallowPartialBareRepoPath(config).toString());
}
Path repoDirectoryPath = getRepoParentFolder(config);
Path repoPath = Paths.get(repoDirectoryPath.toString(), config.getRepoName());

if (SystemUtil.isTestEnvironment() && Files.exists(repoPath) && !shouldFreshClone) {
logger.info("Skipped cloning from " + config.getLocation()
+ " as it was cloned before and cloning is not forced.");
return true;
}

Files.createDirectories(repoDirectoryPath);

Path outputDirectory = Paths.get(repoDirectoryPath.toString(),
Expand Down
30 changes: 25 additions & 5 deletions src/main/java/reposense/report/ReportGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ public class ReportGenerator {
private static final List<String> assetsFilesWhiteList =
Collections.unmodifiableList(Arrays.asList(new String[] {"favicon.ico"}));

private static final boolean DEFAULT_SHOULD_FRESH_CLONE = false;

/**
* Generates the authorship and commits JSON file for each repo in {@code configs} at {@code outputPath}, as
* well as the summary JSON file of all the repos.
Expand All @@ -110,6 +112,23 @@ public static List<Path> generateReposReport(List<RepoConfiguration> configs, St
ReportConfiguration reportConfig, String generationDate, Date cliSinceDate, Date untilDate,
boolean isSinceDateProvided, boolean isUntilDateProvided, int numCloningThreads, int numAnalysisThreads,
Supplier<String> reportGenerationTimeProvider, ZoneId zoneId) throws IOException {
return generateReposReport(configs, outputPath, assetsPath, reportConfig, generationDate,
cliSinceDate, untilDate, isSinceDateProvided, isUntilDateProvided, numCloningThreads,
numAnalysisThreads, reportGenerationTimeProvider, zoneId, DEFAULT_SHOULD_FRESH_CLONE);
}

/**
* Generates the authorship and commits JSON file for each repo in {@code configs} at {@code outputPath}, as
* well as the summary JSON file of all the repos.
*
* @return the list of file paths that were generated.
* @throws IOException if templateZip.zip does not exists in jar file.
*/
public static List<Path> generateReposReport(List<RepoConfiguration> configs, String outputPath, String assetsPath,
ReportConfiguration reportConfig, String generationDate, Date cliSinceDate, Date untilDate,
boolean isSinceDateProvided, boolean isUntilDateProvided, int numCloningThreads, int numAnalysisThreads,
Supplier<String> reportGenerationTimeProvider, ZoneId zoneId,
boolean shouldFreshClone) throws IOException {
prepareTemplateFile(reportConfig, outputPath);
if (Files.exists(Paths.get(assetsPath))) {
FileUtil.copyDirectoryContents(assetsPath, outputPath, assetsFilesWhiteList);
Expand All @@ -119,7 +138,7 @@ public static List<Path> generateReposReport(List<RepoConfiguration> configs, St
progressTracker = new ProgressTracker(configs.size());

List<Path> reportFoldersAndFiles = cloneAndAnalyzeRepos(configs, outputPath,
numCloningThreads, numAnalysisThreads);
numCloningThreads, numAnalysisThreads, shouldFreshClone);

Date reportSinceDate = (cliSinceDate.equals(SinceDateArgumentType.ARBITRARY_FIRST_COMMIT_DATE))
? earliestSinceDate : cliSinceDate;
Expand Down Expand Up @@ -192,7 +211,7 @@ private static Map<RepoLocation, List<RepoConfiguration>> groupConfigsByRepoLoca
* @return A list of paths to the JSON report files generated for each repository.
*/
private static List<Path> cloneAndAnalyzeRepos(List<RepoConfiguration> configs, String outputPath,
int numCloningThreads, int numAnalysisThreads) {
int numCloningThreads, int numAnalysisThreads, boolean shouldFreshClone) {
Map<RepoLocation, List<RepoConfiguration>> repoLocationMap = groupConfigsByRepoLocation(configs);
List<RepoLocation> repoLocationList = new ArrayList<>(repoLocationMap.keySet());

Expand All @@ -208,7 +227,7 @@ private static List<Path> cloneAndAnalyzeRepos(List<RepoConfiguration> configs,
// Note that the `cloneExecutor` is passed as a parameter to ensure that the number of threads used
// for cloning is no more than `numCloningThreads`.
CompletableFuture<CloneJobOutput> cloneFuture = CompletableFuture.supplyAsync(() ->
cloneRepo(configsToAnalyze.get(0), location), cloneExecutor);
cloneRepo(configsToAnalyze.get(0), location, shouldFreshClone), cloneExecutor);

// The `thenApplyAsync` method is used to analyze the cloned repo in parallel.
// This ensures that the analysis job for each repo will only be run after the repo has been cloned.
Expand Down Expand Up @@ -258,9 +277,10 @@ private static List<Path> cloneAndAnalyzeRepos(List<RepoConfiguration> configs,
* @return A {@code CloneJobOutput} object comprising the {@code location} of the repo, whether the cloning was
* successful, and the {@code defaultBranch} of the repo.
*/
private static CloneJobOutput cloneRepo(RepoConfiguration config, RepoLocation location) {
private static CloneJobOutput cloneRepo(RepoConfiguration config, RepoLocation location,
boolean shouldFreshClone) {
RepoCloner repoCloner = new RepoCloner();
repoCloner.cloneBare(config);
repoCloner.cloneBare(config, shouldFreshClone);
RepoLocation clonedRepoLocation = repoCloner.getClonedRepoLocation();
if (clonedRepoLocation != null) {
String defaultBranch = repoCloner.getCurrentRepoDefaultBranch();
Expand Down
Loading

0 comments on commit eab69a9

Please sign in to comment.