diff --git a/commons/src/main/java/io/github/martinwitt/laughing_train/domain/entity/AnalyzerStatus.java b/commons/src/main/java/io/github/martinwitt/laughing_train/domain/entity/AnalyzerStatus.java new file mode 100644 index 000000000..cfee87863 --- /dev/null +++ b/commons/src/main/java/io/github/martinwitt/laughing_train/domain/entity/AnalyzerStatus.java @@ -0,0 +1,71 @@ +package io.github.martinwitt.laughing_train.domain.entity; + +import java.io.Serializable; + +public class AnalyzerStatus implements Serializable { + + private String analyzerName; + private Status status; + private int numberOfIssues; + + AnalyzerStatus(String analyzerName, Status status, int numberOfIssues) { + this.analyzerName = analyzerName; + this.status = status; + this.numberOfIssues = numberOfIssues; + } + + public static AnalyzerStatus success(String analyzerName, int numberOfIssues) { + return new AnalyzerStatus(analyzerName, Status.SUCCESS, numberOfIssues); + } + + public static AnalyzerStatus failure(String analyzerName, int numberOfIssues) { + return new AnalyzerStatus(analyzerName, Status.FAILURE, numberOfIssues); + } + + enum Status { + SUCCESS, + FAILURE + } + + /** + * @return the analyzerName + */ + public String getAnalyzerName() { + return analyzerName; + } + + /** + * @param analyzerName the analyzerName to set + */ + public void setAnalyzerName(String analyzerName) { + this.analyzerName = analyzerName; + } + + /** + * @return the status + */ + public Status getStatus() { + return status; + } + + /** + * @param status the status to set + */ + public void setStatus(Status status) { + this.status = status; + } + + /** + * @return the numberOfIssues + */ + public int getNumberOfIssues() { + return numberOfIssues; + } + + /** + * @param numberOfIssues the numberOfIssues to set + */ + public void setNumberOfIssues(int numberOfIssues) { + this.numberOfIssues = numberOfIssues; + } +} diff --git a/commons/src/main/java/io/github/martinwitt/laughing_train/domain/entity/GitHubCommit.java b/commons/src/main/java/io/github/martinwitt/laughing_train/domain/entity/GitHubCommit.java new file mode 100644 index 000000000..42b9edef2 --- /dev/null +++ b/commons/src/main/java/io/github/martinwitt/laughing_train/domain/entity/GitHubCommit.java @@ -0,0 +1,49 @@ +package io.github.martinwitt.laughing_train.domain.entity; + +import java.io.Serializable; +import java.util.List; + +public class GitHubCommit implements Serializable { + + private String commitHash; + private List analyzerStatuses; + + /** + * @param commitHash + * @param localDateTime + * @param analyzerStatuses + */ + public GitHubCommit(String commitHash, List analyzerStatuses) { + this.commitHash = commitHash; + this.analyzerStatuses = analyzerStatuses; + } + /** + * @return the commitHash + */ + public String getCommitHash() { + return commitHash; + } + /** + * @param commitHash the commitHash to set + */ + public void setCommitHash(String commitHash) { + this.commitHash = commitHash; + } + + /** + * @return the analyzerStatuses + */ + public List getAnalyzerStatuses() { + return analyzerStatuses; + } + /** + * @param analyzerStatuses the analyzerStatuses to set + */ + public void setAnalyzerStatuses(List analyzerStatuses) { + this.analyzerStatuses = analyzerStatuses; + } + + public void addAnalyzerStatus(AnalyzerStatus analyzerStatus) { + this.analyzerStatuses.add(analyzerStatus); + } +} diff --git a/commons/src/main/java/io/github/martinwitt/laughing_train/domain/entity/Project.java b/commons/src/main/java/io/github/martinwitt/laughing_train/domain/entity/Project.java index 5d6da1c72..22ebcd6da 100644 --- a/commons/src/main/java/io/github/martinwitt/laughing_train/domain/entity/Project.java +++ b/commons/src/main/java/io/github/martinwitt/laughing_train/domain/entity/Project.java @@ -10,11 +10,13 @@ public class Project implements Serializable { private String projectName; private String projectUrl; private List commitHashes; + private List commits; public Project(String projectName, String projectUrl) { this.projectName = Objects.requireNonNull(projectName); this.projectUrl = Objects.requireNonNull(projectUrl); commitHashes = new ArrayList<>(); + commits = new ArrayList<>(); } /** @@ -44,6 +46,17 @@ public boolean removeCommitHash(String commitHash) { return commitHashes.remove(commitHash); } + /** + * @return the commits + */ + public List getCommits() { + return commits; + } + + public boolean addCommitHash(GitHubCommit commit) { + return commits.add(commit); + } + /** * @return the commitHashes */ diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/graphql/endpoints/ProjectGraphQL.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/graphql/endpoints/ProjectGraphQL.java index 0512af924..ab202f0b1 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/graphql/endpoints/ProjectGraphQL.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/graphql/endpoints/ProjectGraphQL.java @@ -6,7 +6,7 @@ import io.github.martinwitt.laughing_train.api.graphql.dto.ProjectGraphQLDto; import io.github.martinwitt.laughing_train.domain.entity.Project; import io.github.martinwitt.laughing_train.domain.entity.ProjectConfig; -import io.github.martinwitt.laughing_train.mining.PeriodicMiner; +import io.github.martinwitt.laughing_train.mining.QodanaPeriodicMiner; import io.github.martinwitt.laughing_train.persistence.repository.ProjectConfigRepository; import io.github.martinwitt.laughing_train.persistence.repository.ProjectRepository; import io.github.martinwitt.laughing_train.services.ProjectConfigService; @@ -40,7 +40,7 @@ public class ProjectGraphQL { ProjectConfigRepository projectConfigRepository; @Inject - PeriodicMiner periodicMiner; + QodanaPeriodicMiner periodicMiner; @Query("getProjects") @Description("Gets all projects from the database") diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/QodanaPeriodicMiner.java similarity index 79% rename from github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java rename to github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/QodanaPeriodicMiner.java index db78f8c72..ad2f1fd81 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/QodanaPeriodicMiner.java @@ -3,16 +3,15 @@ import com.google.common.flogger.FluentLogger; import io.github.martinwitt.laughing_train.data.ProjectRequest; import io.github.martinwitt.laughing_train.data.ProjectResult; -import io.github.martinwitt.laughing_train.data.ProjectResult.Success; import io.github.martinwitt.laughing_train.data.QodanaResult; import io.github.martinwitt.laughing_train.data.request.AnalyzerRequest; -import io.github.martinwitt.laughing_train.data.result.CodeAnalyzerResult; import io.github.martinwitt.laughing_train.domain.entity.AnalyzerResult; +import io.github.martinwitt.laughing_train.domain.entity.AnalyzerStatus; +import io.github.martinwitt.laughing_train.domain.entity.GitHubCommit; import io.github.martinwitt.laughing_train.domain.entity.Project; import io.github.martinwitt.laughing_train.persistence.repository.ProjectRepository; import io.github.martinwitt.laughing_train.services.ProjectService; import io.github.martinwitt.laughing_train.services.QodanaService; -import io.github.martinwitt.laughing_train.services.SpoonAnalyzerService; import io.micrometer.core.instrument.MeterRegistry; import io.quarkus.runtime.StartupEvent; import io.vertx.core.Vertx; @@ -30,9 +29,10 @@ import org.kohsuke.github.GitHub; @ApplicationScoped -public class PeriodicMiner { +public class QodanaPeriodicMiner { static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final String ANALYZER_NAME = "Qodana"; final MiningPrinter miningPrinter; final Vertx vertx; @@ -40,21 +40,19 @@ public class PeriodicMiner { final ProjectRepository projectRepository; final QodanaService qodanaService; final ProjectService projectService; - final SpoonAnalyzerService spoonAnalyzerService; MeterRegistry registry; private final Random random = new Random(); private Queue queue = new ArrayDeque<>(); - public PeriodicMiner( + public QodanaPeriodicMiner( MeterRegistry registry, Vertx vertx, SearchProjectService searchProjectService, ProjectRepository projectRepository, QodanaService qodanaService, ProjectService projectService, - MiningPrinter miningPrinter, - SpoonAnalyzerService spoonAnalyzerService) { + MiningPrinter miningPrinter) { this.registry = registry; this.vertx = vertx; this.searchProjectService = searchProjectService; @@ -62,7 +60,6 @@ public PeriodicMiner( this.qodanaService = qodanaService; this.projectService = projectService; this.miningPrinter = miningPrinter; - this.spoonAnalyzerService = spoonAnalyzerService; } private Project getRandomProject() throws IOException { @@ -91,10 +88,9 @@ private void mineRandomRepo() { mineRandomRepo(); return; } - if (checkoutResult instanceof ProjectResult.Success success) { String commitHash = success.project().commitHash(); - if (isAlreadyMined(success, commitHash)) { + if (isAlreadyMined(success, commitHash, ANALYZER_NAME)) { logger.atInfo().log( "Project %s already analyzed with commit hash %s", success.project(), commitHash); tryDeleteProject(success); @@ -102,32 +98,17 @@ private void mineRandomRepo() { } logger.atInfo().log("Successfully checked out project %s", success.project()); var qodanaResult = analyzeProject(success); - var spoonResult = analyzeProjectWithSpoon(success); List results = new ArrayList<>(); if (qodanaResult instanceof QodanaResult.Failure failure) { logger.atWarning().log("Failed to analyze project %s", failure.message()); tryDeleteProject(success); } - if (spoonResult instanceof CodeAnalyzerResult.Failure error) { - logger.atWarning().log("Failed to analyze project with spoon %s", error.message()); - tryDeleteProject(success); - } - if (spoonResult instanceof CodeAnalyzerResult.Success successResult) { - results.addAll(successResult.results()); - } if (qodanaResult instanceof QodanaResult.Success successResult) { logger.atInfo().log("Successfully analyzed project %s", success.project()); - results.addAll(successResult.result()); - } - if (results.isEmpty()) { - logger.atWarning().log("No results for project %s", success.project()); + saveResults(results, project); tryDeleteProject(success); - mineRandomRepo(); - return; } - saveResults(results, project); - addOrUpdateCommitHash(success); - tryDeleteProject(success); + addOrUpdateCommitHash(success, qodanaResult); } } catch (Exception e) { logger.atWarning().withCause(e).log("Failed to mine random repo"); @@ -139,18 +120,6 @@ private void mineRandomRepo() { } } - private CodeAnalyzerResult analyzeProjectWithSpoon(Success success) { - logger.atInfo().log("Analyzing project %s with spoon", success.project()); - CodeAnalyzerResult analyze = spoonAnalyzerService.analyze(new AnalyzerRequest.WithProject(success.project())); - logger.atInfo().log("Successfully analyzed project %s with spoon", success.project()); - return analyze; - } - - private boolean isAlreadyMined(ProjectResult.Success success, String commitHash) { - return projectRepository.findByProjectUrl(success.project().url()).stream() - .anyMatch(it -> !it.getCommitHashes().contains(commitHash)); - } - private ProjectResult checkoutProject(Project project) throws IOException { return projectService.handleProjectRequest(new ProjectRequest.WithUrl(project.getProjectUrl())); } @@ -168,22 +137,6 @@ private void tryDeleteProject(ProjectResult.Success project) { } } - private void addOrUpdateCommitHash(ProjectResult.Success projectResult) { - String name = projectResult.project().name(); - String commitHash = projectResult.project().commitHash(); - var list = projectRepository.findByProjectName(name); - if (list.isEmpty()) { - Project newProject = new Project(name, projectResult.project().url()); - newProject.addCommitHash(commitHash); - projectRepository.create(newProject); - } else { - logger.atInfo().log("Updating commit hash for %s", name); - var oldProject = list.get(0); - oldProject.addCommitHash(commitHash); - projectRepository.save(oldProject); - } - } - private void saveResults(List results, Project project) { try { String content = printFormattedResults(project, results); @@ -227,4 +180,52 @@ public void addToQueue(Project project) { } queue.add(project); } + + private void addOrUpdateCommitHash(ProjectResult.Success projectResult, QodanaResult spoonResult) { + String name = projectResult.project().name(); + String commitHash = projectResult.project().commitHash(); + var list = projectRepository.findByProjectUrl(projectResult.project().url()); + AnalyzerStatus analyzerStatus = getAnalyzerStatus(spoonResult); + if (list.isEmpty()) { + Project newProject = new Project(name, projectResult.project().url()); + newProject.addCommitHash(commitHash); + var commits = newProject.getCommits(); + commits.stream() + .filter(v -> v.getCommitHash().equals(commitHash)) + .findFirst() + .ifPresent(v -> { + v.addAnalyzerStatus(analyzerStatus); + }); + projectRepository.create(newProject); + } else { + logger.atInfo().log("Updating commit hash for %s", name); + var oldProject = list.get(0); + oldProject.addCommitHash(commitHash); + var commits = oldProject.getCommits(); + GitHubCommit gitHubCommit = new GitHubCommit(commitHash, new ArrayList<>()); + commits.add(gitHubCommit); + gitHubCommit.addAnalyzerStatus(analyzerStatus); + oldProject.addCommitHash(gitHubCommit); + projectRepository.save(oldProject); + } + } + + private AnalyzerStatus getAnalyzerStatus(QodanaResult spoonResult) { + AnalyzerStatus analyzerStatus = null; + if (spoonResult instanceof QodanaResult.Success success) { + analyzerStatus = + AnalyzerStatus.success(ANALYZER_NAME, success.result().size()); + } else if (spoonResult instanceof QodanaResult.Failure failure) { + analyzerStatus = AnalyzerStatus.failure(ANALYZER_NAME, 0); + } + return analyzerStatus; + } + + private boolean isAlreadyMined(ProjectResult.Success success, String commitHash, String analyzerName) { + return projectRepository.findByProjectUrl(success.project().url()).stream() + .flatMap(v -> v.getCommits().stream()) + .filter(v -> v.getCommitHash().equals(commitHash)) + .flatMap(v -> v.getAnalyzerStatuses().stream()) + .anyMatch(v -> v.getAnalyzerName().equals(analyzerName)); + } } diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/SpoonPeriodicMiner.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/SpoonPeriodicMiner.java new file mode 100644 index 000000000..f5fc1cf8d --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/SpoonPeriodicMiner.java @@ -0,0 +1,185 @@ +package io.github.martinwitt.laughing_train.mining; + +import com.google.common.flogger.FluentLogger; +import io.github.martinwitt.laughing_train.data.ProjectRequest; +import io.github.martinwitt.laughing_train.data.ProjectResult; +import io.github.martinwitt.laughing_train.data.ProjectResult.Success; +import io.github.martinwitt.laughing_train.data.request.AnalyzerRequest; +import io.github.martinwitt.laughing_train.data.result.CodeAnalyzerResult; +import io.github.martinwitt.laughing_train.domain.entity.AnalyzerStatus; +import io.github.martinwitt.laughing_train.domain.entity.GitHubCommit; +import io.github.martinwitt.laughing_train.domain.entity.Project; +import io.github.martinwitt.laughing_train.persistence.repository.ProjectRepository; +import io.github.martinwitt.laughing_train.services.ProjectService; +import io.github.martinwitt.laughing_train.services.QodanaService; +import io.github.martinwitt.laughing_train.services.SpoonAnalyzerService; +import io.micrometer.core.instrument.MeterRegistry; +import io.quarkus.runtime.StartupEvent; +import io.vertx.core.Vertx; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Queue; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.apache.commons.io.FileUtils; + +@ApplicationScoped +public abstract class SpoonPeriodicMiner { + + static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final String ANALYZER_NAME = "spoon-analyzer"; + final MiningPrinter miningPrinter; + final Vertx vertx; + final SearchProjectService searchProjectService; + final ProjectRepository projectRepository; + final ProjectService projectService; + MeterRegistry registry; + final SpoonAnalyzerService spoonAnalyzerService; + private final Random random = new Random(); + private Queue queue = new ArrayDeque<>(); + + public SpoonPeriodicMiner( + MeterRegistry registry, + Vertx vertx, + SearchProjectService searchProjectService, + ProjectRepository projectRepository, + QodanaService qodanaService, + ProjectService projectService, + MiningPrinter miningPrinter, + SpoonAnalyzerService spoonAnalyzerService) { + this.registry = registry; + this.vertx = vertx; + this.searchProjectService = searchProjectService; + this.projectRepository = projectRepository; + this.projectService = projectService; + this.miningPrinter = miningPrinter; + this.spoonAnalyzerService = spoonAnalyzerService; + } + + private Project getRandomProject() throws IOException { + if (random.nextBoolean()) { + return searchProjectService.searchProjectOnGithub(); + } else { + return getKnownProject(); + } + } + + private Project getKnownProject() { + var list = projectRepository.getAll(); + return list.get(random.nextInt(list.size())); + } + + void mine(@Observes StartupEvent event) { + vertx.setTimer(TimeUnit.MINUTES.toMillis(5), v -> vertx.executeBlocking(it -> mineRandomRepo())); + } + + private void mineRandomRepo() { + try { + var project = queue.isEmpty() ? getRandomProject() : queue.poll(); + var checkoutResult = checkoutProject(project); + if (checkoutResult instanceof ProjectResult.Error) { + logger.atWarning().log("Failed to checkout project %s", project); + mineRandomRepo(); + return; + } + + if (checkoutResult instanceof ProjectResult.Success success) { + String commitHash = success.project().commitHash(); + if (isAlreadyMined(success, commitHash, ANALYZER_NAME)) { + logger.atInfo().log( + "Project %s already analyzed with commit hash %s", success.project(), commitHash); + tryDeleteProject(success); + mineRandomRepo(); + } + logger.atInfo().log("Successfully checked out project %s", success.project()); + var spoonResult = analyzeProjectWithSpoon(success); + if (spoonResult instanceof CodeAnalyzerResult.Failure error) { + logger.atWarning().log("Failed to analyze project with spoon %s", error.message()); + tryDeleteProject(success); + } + if (spoonResult instanceof CodeAnalyzerResult.Success successResult) { + addOrUpdateCommitHash(success, spoonResult); + } + tryDeleteProject(success); + } + } catch (Exception e) { + logger.atWarning().withCause(e).log("Failed to mine random repo"); + registry.counter("mining.error").increment(); + } finally { + logger.atInfo().log("Queue size: %s", queue.size()); + logger.atInfo().log("Mining next repo in 1 minute"); + vertx.setTimer(TimeUnit.MINUTES.toMillis(1), v -> vertx.executeBlocking(it -> mineRandomRepo())); + } + } + + private CodeAnalyzerResult analyzeProjectWithSpoon(Success success) { + logger.atInfo().log("Analyzing project %s with spoon", success.project()); + CodeAnalyzerResult analyze = spoonAnalyzerService.analyze(new AnalyzerRequest.WithProject(success.project())); + logger.atInfo().log("Successfully analyzed project %s with spoon", success.project()); + return analyze; + } + + private boolean isAlreadyMined(ProjectResult.Success success, String commitHash, String analyzerName) { + return projectRepository.findByProjectUrl(success.project().url()).stream() + .flatMap(v -> v.getCommits().stream()) + .filter(v -> v.getCommitHash().equals(commitHash)) + .flatMap(v -> v.getAnalyzerStatuses().stream()) + .anyMatch(v -> v.getAnalyzerName().equals(analyzerName)); + } + + private ProjectResult checkoutProject(Project project) throws IOException { + return projectService.handleProjectRequest(new ProjectRequest.WithUrl(project.getProjectUrl())); + } + + private void tryDeleteProject(ProjectResult.Success project) { + try { + FileUtils.deleteDirectory(project.project().folder()); + } catch (IOException e) { + logger.atWarning().log( + "Failed to delete project " + project.project().folder()); + } + } + + private void addOrUpdateCommitHash(ProjectResult.Success projectResult, CodeAnalyzerResult spoonResult) { + String name = projectResult.project().name(); + String commitHash = projectResult.project().commitHash(); + var list = projectRepository.findByProjectUrl(projectResult.project().url()); + AnalyzerStatus analyzerStatus = getAnalyzerStatus(spoonResult); + if (list.isEmpty()) { + Project newProject = new Project(name, projectResult.project().url()); + newProject.addCommitHash(commitHash); + var commits = newProject.getCommits(); + commits.stream() + .filter(v -> v.getCommitHash().equals(commitHash)) + .findFirst() + .ifPresent(v -> { + v.addAnalyzerStatus(analyzerStatus); + }); + projectRepository.create(newProject); + } else { + logger.atInfo().log("Updating commit hash for %s", name); + var oldProject = list.get(0); + oldProject.addCommitHash(commitHash); + var commits = oldProject.getCommits(); + GitHubCommit gitHubCommit = new GitHubCommit(commitHash, new ArrayList<>()); + commits.add(gitHubCommit); + gitHubCommit.addAnalyzerStatus(analyzerStatus); + oldProject.addCommitHash(gitHubCommit); + projectRepository.save(oldProject); + } + } + + private AnalyzerStatus getAnalyzerStatus(CodeAnalyzerResult spoonResult) { + AnalyzerStatus analyzerStatus = null; + if (spoonResult instanceof CodeAnalyzerResult.Success success) { + analyzerStatus = + AnalyzerStatus.success(ANALYZER_NAME, success.results().size()); + } else if (spoonResult instanceof CodeAnalyzerResult.Failure failure) { + analyzerStatus = AnalyzerStatus.failure(ANALYZER_NAME, 0); + } + return analyzerStatus; + } +}