diff --git a/buildSrc/src/main/groovy/xyz.keksdose.spoon.code_solver.java-common-conventions.gradle b/buildSrc/src/main/groovy/xyz.keksdose.spoon.code_solver.java-common-conventions.gradle index 3db6ddacb..78d3d78e6 100644 --- a/buildSrc/src/main/groovy/xyz.keksdose.spoon.code_solver.java-common-conventions.gradle +++ b/buildSrc/src/main/groovy/xyz.keksdose.spoon.code_solver.java-common-conventions.gradle @@ -13,7 +13,7 @@ dependencies { // Use JUnit Jupiter for testing. testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3' implementation 'com.google.guava:guava:32.1.1-jre' - implementation 'fr.inria.gforge.spoon:spoon-core:+' + implementation 'fr.inria.gforge.spoon:spoon-core:10.4.0' implementation 'com.google.flogger:flogger:0.7.4' implementation 'com.google.flogger:flogger-system-backend:0.7.4' implementation group: 'ch.qos.logback', name: 'logback-core', version: '1.4.8' diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/graphql/endpoints/ProjectGraphQL.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/graphql/endpoints/ProjectGraphQL.java index ab202f0b1..d4b6a57d2 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/graphql/endpoints/ProjectGraphQL.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/graphql/endpoints/ProjectGraphQL.java @@ -74,12 +74,8 @@ public ProjectGraphQLDto addProject(String projectUrl, String projectName) { if (!projectRepository.existsByProjectUrl(projectUrl)) { logger.atInfo().log("Project does not exist yet, creating it"); Project project = new Project(projectUrl, projectName); - periodicMiner.addToQueue(project); return mapToDto(projectRepository.create(project)); } else { - periodicMiner.addToQueue( - projectRepository.findByProjectUrl(projectUrl).get(0)); - logger.atInfo().log("Project %s already exists", projectName); throw new RuntimeException("Project already exists"); } diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/AnalyzerResultsPersistence.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/AnalyzerResultsPersistence.java new file mode 100644 index 000000000..c8c1db4cb --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/AnalyzerResultsPersistence.java @@ -0,0 +1,82 @@ +package io.github.martinwitt.laughing_train.mining; + +import com.google.common.flogger.FluentLogger; +import io.github.martinwitt.laughing_train.data.Project; +import io.github.martinwitt.laughing_train.data.result.CodeAnalyzerResult; +import io.github.martinwitt.laughing_train.domain.entity.AnalyzerStatus; +import io.github.martinwitt.laughing_train.domain.entity.GitHubCommit; +import io.github.martinwitt.laughing_train.mining.requests.MineNextProject; +import io.github.martinwitt.laughing_train.mining.requests.StoreResults; +import io.github.martinwitt.laughing_train.persistence.repository.ProjectRepository; +import io.quarkus.vertx.ConsumeEvent; +import io.vertx.core.eventbus.EventBus; +import java.util.ArrayList; +import java.util.List; + +public class AnalyzerResultsPersistence { + + public static final String SERVICE_NAME = "analyzerResultsPersistence"; + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + ProjectRepository projectRepository; + EventBus eventBus; + + public AnalyzerResultsPersistence(ProjectRepository projectRepository, EventBus eventBus) { + this.projectRepository = projectRepository; + this.eventBus = eventBus; + } + + @ConsumeEvent(value = SERVICE_NAME, blocking = true) + void persistResults(StoreResults storeResults) { + Project project = storeResults.project(); + CodeAnalyzerResult result = storeResults.result(); + addOrUpdateCommitHash(project, result, SERVICE_NAME); + if (result instanceof CodeAnalyzerResult.Failure failure) { + logger.atInfo().log("Analyzer %s failed for project %s", SERVICE_NAME, project.name()); + + } else if (result instanceof CodeAnalyzerResult.Success success) { + logger.atInfo().log("Analyzer %s succeeded for project %s", SERVICE_NAME, project.name()); + } + eventBus.publish("miner", new MineNextProject(storeResults.analyzerName())); + } + + private AnalyzerStatus getAnalyzerStatus(CodeAnalyzerResult spoonResult, String name) { + AnalyzerStatus analyzerStatus = null; + if (spoonResult instanceof CodeAnalyzerResult.Success success) { + analyzerStatus = AnalyzerStatus.success(name, success.results().size()); + } else if (spoonResult instanceof CodeAnalyzerResult.Failure failure) { + analyzerStatus = AnalyzerStatus.failure(name, 0); + } + return analyzerStatus; + } + + private void addOrUpdateCommitHash(Project project, CodeAnalyzerResult spoonResult, String analyzerName) { + String name = project.name(); + String commitHash = project.commitHash(); + List list = + projectRepository.findByProjectUrl(project.url()); + AnalyzerStatus analyzerStatus = getAnalyzerStatus(spoonResult, analyzerName); + if (list.isEmpty()) { + io.github.martinwitt.laughing_train.domain.entity.Project newProject = + new io.github.martinwitt.laughing_train.domain.entity.Project(name, project.url()); + newProject.addCommitHash(commitHash); + var commits = newProject.getCommits(); + commits.stream() + .filter(v -> v.getCommitHash().equals(commitHash)) + .findFirst() + .ifPresent(v -> { + v.addAnalyzerStatus(analyzerStatus); + }); + projectRepository.create(newProject); + } else { + logger.atInfo().log("Updating commit hash for %s", name); + var oldProject = list.get(0); + oldProject.addCommitHash(commitHash); + var commits = oldProject.getCommits(); + GitHubCommit gitHubCommit = new GitHubCommit(commitHash, new ArrayList<>()); + commits.add(gitHubCommit); + gitHubCommit.addAnalyzerStatus(analyzerStatus); + oldProject.addCommitHash(gitHubCommit); + projectRepository.save(oldProject); + } + } +} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/CodeMiner.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/CodeMiner.java new file mode 100644 index 000000000..e10ff17e3 --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/CodeMiner.java @@ -0,0 +1,6 @@ +package io.github.martinwitt.laughing_train.mining; + +public interface CodeMiner { + + String getAnalyzerName(); +} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/MiningStartup.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/MiningStartup.java new file mode 100644 index 000000000..0dfa6e089 --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/MiningStartup.java @@ -0,0 +1,20 @@ +package io.github.martinwitt.laughing_train.mining; + +import io.github.martinwitt.laughing_train.mining.requests.MineNextProject; +import io.quarkus.runtime.StartupEvent; +import io.vertx.mutiny.core.Vertx; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import jakarta.inject.Inject; + +@ApplicationScoped +public class MiningStartup { + + @Inject + Vertx vertx; + + void startup(@Observes StartupEvent event) { + vertx.eventBus().send("miner", new MineNextProject(QodanaPeriodicMiner.ANALYZER_NAME)); + vertx.eventBus().send("miner", new MineNextProject(SpoonPeriodicMiner.ANALYZER_NAME)); + } +} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/ProjectSupplier.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/ProjectSupplier.java new file mode 100644 index 000000000..c7b8ac5b4 --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/ProjectSupplier.java @@ -0,0 +1,57 @@ +package io.github.martinwitt.laughing_train.mining; + +import io.github.martinwitt.laughing_train.data.ProjectRequest; +import io.github.martinwitt.laughing_train.data.ProjectResult; +import io.github.martinwitt.laughing_train.domain.entity.Project; +import io.github.martinwitt.laughing_train.mining.requests.GetProject; +import io.github.martinwitt.laughing_train.persistence.repository.ProjectRepository; +import io.github.martinwitt.laughing_train.services.ProjectService; +import io.quarkus.vertx.ConsumeEvent; +import java.io.IOException; +import java.util.Random; + +public class ProjectSupplier { + + public static final String SERVICE_NAME = "projectSupplier"; + + final SearchProjectService searchProjectService; + final ProjectRepository projectRepository; + final ProjectService projectService; + private static final Random random = new Random(); + + ProjectSupplier( + SearchProjectService searchProjectService, + ProjectRepository projectRepository, + ProjectService projectService) { + this.searchProjectService = searchProjectService; + this.projectRepository = projectRepository; + this.projectService = projectService; + } + + @ConsumeEvent(value = SERVICE_NAME, blocking = true) + ProjectResult supplyProject(GetProject getProject) { + try { + Project project = getRandomProject(); + return checkoutProject(project); + } catch (IOException e) { + return new ProjectResult.Error(e.getMessage()); + } + } + + private ProjectResult checkoutProject(Project project) throws IOException { + return projectService.handleProjectRequest(new ProjectRequest.WithUrl(project.getProjectUrl())); + } + + private Project getRandomProject() throws IOException { + if (random.nextBoolean()) { + return searchProjectService.searchProjectOnGithub(); + } else { + return getKnownProject(); + } + } + + private Project getKnownProject() { + var list = projectRepository.getAll(); + return list.get(random.nextInt(list.size())); + } +} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/QodanaPeriodicMiner.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/QodanaPeriodicMiner.java index 510d8bf2f..ecedd56c6 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/QodanaPeriodicMiner.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/QodanaPeriodicMiner.java @@ -1,139 +1,39 @@ package io.github.martinwitt.laughing_train.mining; import com.google.common.flogger.FluentLogger; -import io.github.martinwitt.laughing_train.data.ProjectRequest; import io.github.martinwitt.laughing_train.data.ProjectResult; import io.github.martinwitt.laughing_train.data.QodanaResult; import io.github.martinwitt.laughing_train.data.request.AnalyzerRequest; -import io.github.martinwitt.laughing_train.domain.entity.AnalyzerResult; -import io.github.martinwitt.laughing_train.domain.entity.AnalyzerStatus; -import io.github.martinwitt.laughing_train.domain.entity.GitHubCommit; -import io.github.martinwitt.laughing_train.domain.entity.Project; +import io.github.martinwitt.laughing_train.data.result.CodeAnalyzerResult; +import io.github.martinwitt.laughing_train.mining.requests.GetProject; +import io.github.martinwitt.laughing_train.mining.requests.MineNextProject; +import io.github.martinwitt.laughing_train.mining.requests.StoreResults; import io.github.martinwitt.laughing_train.persistence.repository.ProjectRepository; -import io.github.martinwitt.laughing_train.services.ProjectService; import io.github.martinwitt.laughing_train.services.QodanaService; -import io.micrometer.core.instrument.MeterRegistry; import io.quarkus.arc.Unremovable; -import io.quarkus.runtime.StartupEvent; +import io.quarkus.vertx.ConsumeEvent; +import io.vertx.core.Future; import io.vertx.core.Vertx; +import io.vertx.core.eventbus.Message; import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.event.Observes; import java.io.IOException; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.Queue; -import java.util.Random; -import java.util.concurrent.TimeUnit; import org.apache.commons.io.FileUtils; -import org.kohsuke.github.GHRepository; -import org.kohsuke.github.GitHub; @Unremovable @ApplicationScoped public class QodanaPeriodicMiner { static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final String ANALYZER_NAME = "Qodana"; + public static final String ANALYZER_NAME = "Qodana"; - final MiningPrinter miningPrinter; final Vertx vertx; - final SearchProjectService searchProjectService; final ProjectRepository projectRepository; final QodanaService qodanaService; - final ProjectService projectService; - MeterRegistry registry; - final SpoonPeriodicMiner spoonPeriodicMiner; - private final Random random = new Random(); - private Queue queue = new ArrayDeque<>(); - - public QodanaPeriodicMiner( - MeterRegistry registry, - Vertx vertx, - SearchProjectService searchProjectService, - ProjectRepository projectRepository, - QodanaService qodanaService, - ProjectService projectService, - MiningPrinter miningPrinter, - SpoonPeriodicMiner spoonPeriodicMiner) { - this.registry = registry; + public QodanaPeriodicMiner(Vertx vertx, ProjectRepository projectRepository, QodanaService qodanaService) { this.vertx = vertx; - this.searchProjectService = searchProjectService; this.projectRepository = projectRepository; this.qodanaService = qodanaService; - this.projectService = projectService; - this.miningPrinter = miningPrinter; - this.spoonPeriodicMiner = spoonPeriodicMiner; - } - - private Project getRandomProject() throws IOException { - if (random.nextBoolean()) { - return searchProjectService.searchProjectOnGithub(); - } else { - return getKnownProject(); - } - } - - private Project getKnownProject() { - var list = projectRepository.getAll(); - return list.get(random.nextInt(list.size())); - } - - void mine(@Observes StartupEvent event) { - try { - logger.atInfo().log("Starting Qodana periodic miner"); - vertx.exceptionHandler(it -> logger.atWarning().withCause(it).log("Exception in vertx")); - vertx.setTimer(TimeUnit.MINUTES.toMillis(3), v -> vertx.executeBlocking(it -> mineRandomRepo())); - } catch (Exception e) { - logger.atWarning().withCause(e).log("Failed to repo with Qodana"); - } - } - - private void mineRandomRepo() { - try { - logger.atInfo().log("Mining repository with qodana"); - var project = queue.isEmpty() ? getRandomProject() : queue.poll(); - var checkoutResult = checkoutProject(project); - if (checkoutResult instanceof ProjectResult.Error) { - logger.atWarning().log("Failed to checkout project %s", project); - mineRandomRepo(); - return; - } - if (checkoutResult instanceof ProjectResult.Success success) { - String commitHash = success.project().commitHash(); - spoonPeriodicMiner.mineRandomRepo(success); - if (isAlreadyMined(success, commitHash, ANALYZER_NAME)) { - logger.atInfo().log( - "Project %s already analyzed with commit hash %s", success.project(), commitHash); - tryDeleteProject(success); - mineRandomRepo(); - } - logger.atInfo().log("Successfully checked out project %s", success.project()); - var qodanaResult = analyzeProject(success); - List results = new ArrayList<>(); - if (qodanaResult instanceof QodanaResult.Failure failure) { - logger.atWarning().log("Failed to analyze project %s", failure.message()); - tryDeleteProject(success); - } - if (qodanaResult instanceof QodanaResult.Success successResult) { - logger.atInfo().log("Successfully analyzed project %s", success.project()); - saveResults(results, project); - tryDeleteProject(success); - } - addOrUpdateCommitHash(success, qodanaResult); - } - } catch (Exception e) { - logger.atWarning().withCause(e).log("Failed to mine random repo"); - registry.counter("mining.error").increment(); - } finally { - logger.atInfo().log("Mining next repo in 1 minute"); - vertx.setTimer(TimeUnit.MINUTES.toMillis(1), v -> vertx.executeBlocking(it -> mineRandomRepo())); - } - } - - private ProjectResult checkoutProject(Project project) throws IOException { - return projectService.handleProjectRequest(new ProjectRequest.WithUrl(project.getProjectUrl())); } private QodanaResult analyzeProject(ProjectResult.Success message) { @@ -149,95 +49,45 @@ private void tryDeleteProject(ProjectResult.Success project) { } } - private void saveResults(List results, Project project) { - try { - String content = printFormattedResults(project, results); - var laughingRepo = getLaughingRepo(); - updateOrCreateContent(laughingRepo, project.getProjectName(), content); - } catch (Exception e) { - logger.atSevere().withCause(e).log("Error while updating content"); - } - } - - private String printFormattedResults(Project project, List results) { - return "# %s %n %s".formatted(project.getProjectName(), miningPrinter.printAllResults(results)); - } - - private GHRepository getLaughingRepo() throws IOException { - return GitHub.connectUsingOAuth(System.getenv("GITHUB_TOKEN")).getRepository("MartinWitt/laughing-train"); - } - - private void updateOrCreateContent(GHRepository repo, String repoName, String content) { - try { - repo.getFileContent("mining/" + repoName + ".md", "gh-mining") - .update(content, "Update mining results for " + repoName, "gh-mining"); - } catch (Exception ignored) { - // this exception is thrown if the file does not exist yet for new projects - try { - repo.createContent() - .content(content) - .path("mining/" + repoName + ".md") - .message("mining " + repoName) - .branch("gh-mining") - .commit(); - } catch (Exception e) { - logger.atSevere().withCause(e).log("Error while creating mining file"); - } - } - } - - public void addToQueue(Project project) { - if (project == null) { + @ConsumeEvent(value = "miner", blocking = true) + void mineWithQodana(MineNextProject event) { + if (!event.analyzerName().equals(ANALYZER_NAME)) { return; } - queue.add(project); - } - - private void addOrUpdateCommitHash(ProjectResult.Success projectResult, QodanaResult spoonResult) { - String name = projectResult.project().name(); - String commitHash = projectResult.project().commitHash(); - var list = projectRepository.findByProjectUrl(projectResult.project().url()); - AnalyzerStatus analyzerStatus = getAnalyzerStatus(spoonResult); - if (list.isEmpty()) { - Project newProject = new Project(name, projectResult.project().url()); - newProject.addCommitHash(commitHash); - var commits = newProject.getCommits(); - commits.stream() - .filter(v -> v.getCommitHash().equals(commitHash)) - .findFirst() - .ifPresent(v -> { - v.addAnalyzerStatus(analyzerStatus); - }); - projectRepository.create(newProject); - } else { - logger.atInfo().log("Updating commit hash for %s", name); - var oldProject = list.get(0); - oldProject.addCommitHash(commitHash); - var commits = oldProject.getCommits(); - GitHubCommit gitHubCommit = new GitHubCommit(commitHash, new ArrayList<>()); - commits.add(gitHubCommit); - gitHubCommit.addAnalyzerStatus(analyzerStatus); - oldProject.addCommitHash(gitHubCommit); - projectRepository.save(oldProject); - } - } - - private AnalyzerStatus getAnalyzerStatus(QodanaResult spoonResult) { - AnalyzerStatus analyzerStatus = null; - if (spoonResult instanceof QodanaResult.Success success) { - analyzerStatus = - AnalyzerStatus.success(ANALYZER_NAME, success.result().size()); - } else if (spoonResult instanceof QodanaResult.Failure failure) { - analyzerStatus = AnalyzerStatus.failure(ANALYZER_NAME, 0); - } - return analyzerStatus; - } - - private boolean isAlreadyMined(ProjectResult.Success success, String commitHash, String analyzerName) { - return projectRepository.findByProjectUrl(success.project().url()).stream() - .flatMap(v -> v.getCommits().stream()) - .filter(v -> v.getCommitHash().equals(commitHash)) - .flatMap(v -> v.getAnalyzerStatuses().stream()) - .anyMatch(v -> v.getAnalyzerName().equals(analyzerName)); + logger.atInfo().log("Start mining with qodana"); + Future> request = + vertx.eventBus().request(ProjectSupplier.SERVICE_NAME, new GetProject(ANALYZER_NAME)); + request.onSuccess(v -> { + if (v.body() instanceof ProjectResult.Success success) { + var qodanaResult = analyzeProject(success); + if (qodanaResult instanceof QodanaResult.Success qodanaSuccess) { + storeSuccess(success, qodanaSuccess); + } else { + if (qodanaResult instanceof QodanaResult.Failure error) { + storeFailure(success, error); + } + } + } + }); + request.onFailure(v -> { + logger.atWarning().withCause(v).log("Failed to get project"); + }); + } + + private void storeFailure(ProjectResult.Success success, QodanaResult.Failure error) { + logger.atWarning().log("Failed to analyze project with qodana %s", error.message()); + tryDeleteProject(success); + StoreResults storeResults = + new StoreResults(success.project(), new CodeAnalyzerResult.Failure(error.message()), ANALYZER_NAME); + vertx.eventBus().send(AnalyzerResultsPersistence.SERVICE_NAME, storeResults); + } + + private void storeSuccess(ProjectResult.Success success, QodanaResult.Success qodanaSuccess) { + logger.atInfo().log("Successfully analyzed project %s with qodana", success.project()); + StoreResults storeResults = new StoreResults( + success.project(), + new CodeAnalyzerResult.Success(qodanaSuccess.result(), success.project()), + ANALYZER_NAME); + vertx.eventBus().send(AnalyzerResultsPersistence.SERVICE_NAME, storeResults); } } diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/SpoonPeriodicMiner.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/SpoonPeriodicMiner.java index 5a9ed2fb4..f7922daca 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/SpoonPeriodicMiner.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/SpoonPeriodicMiner.java @@ -5,20 +5,17 @@ import io.github.martinwitt.laughing_train.data.ProjectResult.Success; import io.github.martinwitt.laughing_train.data.request.AnalyzerRequest; import io.github.martinwitt.laughing_train.data.result.CodeAnalyzerResult; -import io.github.martinwitt.laughing_train.domain.entity.AnalyzerStatus; -import io.github.martinwitt.laughing_train.domain.entity.GitHubCommit; -import io.github.martinwitt.laughing_train.domain.entity.Project; -import io.github.martinwitt.laughing_train.persistence.repository.ProjectRepository; -import io.github.martinwitt.laughing_train.services.ProjectService; +import io.github.martinwitt.laughing_train.mining.requests.GetProject; +import io.github.martinwitt.laughing_train.mining.requests.MineNextProject; +import io.github.martinwitt.laughing_train.mining.requests.StoreResults; import io.github.martinwitt.laughing_train.services.SpoonAnalyzerService; -import io.micrometer.core.instrument.MeterRegistry; import io.quarkus.arc.Unremovable; +import io.quarkus.vertx.ConsumeEvent; +import io.vertx.core.Future; import io.vertx.core.Vertx; +import io.vertx.core.eventbus.Message; import jakarta.enterprise.context.ApplicationScoped; import java.io.IOException; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Queue; import org.apache.commons.io.FileUtils; @Unremovable @@ -26,57 +23,15 @@ public class SpoonPeriodicMiner { static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final String ANALYZER_NAME = "spoon-analyzer"; - final MiningPrinter miningPrinter; + public static final String ANALYZER_NAME = "spoon-analyzer"; final Vertx vertx; - final SearchProjectService searchProjectService; - final ProjectRepository projectRepository; - final ProjectService projectService; - MeterRegistry registry; final SpoonAnalyzerService spoonAnalyzerService; - private Queue queue = new ArrayDeque<>(); - public SpoonPeriodicMiner( - MeterRegistry registry, - Vertx vertx, - SearchProjectService searchProjectService, - ProjectRepository projectRepository, - ProjectService projectService, - MiningPrinter miningPrinter, - SpoonAnalyzerService spoonAnalyzerService) { - this.registry = registry; + public SpoonPeriodicMiner(Vertx vertx, SpoonAnalyzerService spoonAnalyzerService) { this.vertx = vertx; - this.searchProjectService = searchProjectService; - this.projectRepository = projectRepository; - this.projectService = projectService; - this.miningPrinter = miningPrinter; this.spoonAnalyzerService = spoonAnalyzerService; } - void mineRandomRepo(ProjectResult.Success success) { - try { - logger.atInfo().log("Start mining with spoon"); - String commitHash = success.project().commitHash(); - if (isAlreadyMined(success, commitHash, ANALYZER_NAME)) { - logger.atInfo().log("Project %s already analyzed with commit hash %s", success.project(), commitHash); - } - logger.atInfo().log("Successfully checked out project %s", success.project()); - var spoonResult = analyzeProjectWithSpoon(success); - if (spoonResult instanceof CodeAnalyzerResult.Failure error) { - logger.atWarning().log("Failed to analyze project with spoon %s", error.message()); - tryDeleteProject(success); - } - if (spoonResult instanceof CodeAnalyzerResult.Success successResult) { - addOrUpdateCommitHash(success, spoonResult); - } - } catch (Exception e) { - logger.atWarning().withCause(e).log("Failed to mine random repo"); - registry.counter("mining.error").increment(); - } finally { - logger.atInfo().log("Finished mining with spoon"); - } - } - private CodeAnalyzerResult analyzeProjectWithSpoon(Success success) { logger.atInfo().log("Analyzing project %s with spoon", success.project()); CodeAnalyzerResult analyze = spoonAnalyzerService.analyze(new AnalyzerRequest.WithProject(success.project())); @@ -84,14 +39,6 @@ private CodeAnalyzerResult analyzeProjectWithSpoon(Success success) { return analyze; } - private boolean isAlreadyMined(ProjectResult.Success success, String commitHash, String analyzerName) { - return projectRepository.findByProjectUrl(success.project().url()).stream() - .flatMap(v -> v.getCommits().stream()) - .filter(v -> v.getCommitHash().equals(commitHash)) - .flatMap(v -> v.getAnalyzerStatuses().stream()) - .anyMatch(v -> v.getAnalyzerName().equals(analyzerName)); - } - private void tryDeleteProject(ProjectResult.Success project) { try { FileUtils.deleteDirectory(project.project().folder()); @@ -101,43 +48,45 @@ private void tryDeleteProject(ProjectResult.Success project) { } } - private void addOrUpdateCommitHash(ProjectResult.Success projectResult, CodeAnalyzerResult spoonResult) { - String name = projectResult.project().name(); - String commitHash = projectResult.project().commitHash(); - var list = projectRepository.findByProjectUrl(projectResult.project().url()); - AnalyzerStatus analyzerStatus = getAnalyzerStatus(spoonResult); - if (list.isEmpty()) { - Project newProject = new Project(name, projectResult.project().url()); - newProject.addCommitHash(commitHash); - var commits = newProject.getCommits(); - commits.stream() - .filter(v -> v.getCommitHash().equals(commitHash)) - .findFirst() - .ifPresent(v -> { - v.addAnalyzerStatus(analyzerStatus); - }); - projectRepository.create(newProject); - } else { - logger.atInfo().log("Updating commit hash for %s", name); - var oldProject = list.get(0); - oldProject.addCommitHash(commitHash); - var commits = oldProject.getCommits(); - GitHubCommit gitHubCommit = new GitHubCommit(commitHash, new ArrayList<>()); - commits.add(gitHubCommit); - gitHubCommit.addAnalyzerStatus(analyzerStatus); - oldProject.addCommitHash(gitHubCommit); - projectRepository.save(oldProject); + @ConsumeEvent(value = "miner", blocking = true) + void mineWithSpoon(MineNextProject event) { + if (!event.analyzerName().equals(ANALYZER_NAME)) { + return; } + logger.atInfo().log("Start mining with spoon"); + Future> request = + vertx.eventBus().request(ProjectSupplier.SERVICE_NAME, new GetProject(ANALYZER_NAME)); + request.onSuccess(v -> { + if (v.body() instanceof ProjectResult.Success success) { + var spoonResult = analyzeProjectWithSpoon(success); + if (spoonResult instanceof CodeAnalyzerResult.Success spoonSuccess) { + storeSuccess(success, spoonSuccess); + } else { + if (spoonResult instanceof CodeAnalyzerResult.Failure error) { + storeFailure(success, error); + } + } + } + }); + request.onFailure(v -> { + logger.atWarning().withCause(v).log("Failed to get project"); + }); } - private AnalyzerStatus getAnalyzerStatus(CodeAnalyzerResult spoonResult) { - AnalyzerStatus analyzerStatus = null; - if (spoonResult instanceof CodeAnalyzerResult.Success success) { - analyzerStatus = - AnalyzerStatus.success(ANALYZER_NAME, success.results().size()); - } else if (spoonResult instanceof CodeAnalyzerResult.Failure failure) { - analyzerStatus = AnalyzerStatus.failure(ANALYZER_NAME, 0); - } - return analyzerStatus; + private void storeFailure(ProjectResult.Success success, CodeAnalyzerResult.Failure error) { + logger.atWarning().log("Failed to analyze project with spoon %s", error.message()); + tryDeleteProject(success); + StoreResults storeResults = + new StoreResults(success.project(), new CodeAnalyzerResult.Failure(error.message()), ANALYZER_NAME); + vertx.eventBus().send(AnalyzerResultsPersistence.SERVICE_NAME, storeResults); + } + + private void storeSuccess(ProjectResult.Success success, CodeAnalyzerResult.Success spoonSuccess) { + logger.atInfo().log("Successfully analyzed project %s with spoon", success.project()); + StoreResults storeResults = new StoreResults( + success.project(), + new CodeAnalyzerResult.Success(spoonSuccess.results(), success.project()), + ANALYZER_NAME); + vertx.eventBus().send(AnalyzerResultsPersistence.SERVICE_NAME, storeResults); } } diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/requests/GetProject.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/requests/GetProject.java new file mode 100644 index 000000000..143ea7ea7 --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/requests/GetProject.java @@ -0,0 +1,5 @@ +package io.github.martinwitt.laughing_train.mining.requests; + +import java.io.Serializable; + +public record GetProject(String analyzerName) implements Serializable {} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/requests/MineNextProject.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/requests/MineNextProject.java new file mode 100644 index 000000000..670d2419b --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/requests/MineNextProject.java @@ -0,0 +1,3 @@ +package io.github.martinwitt.laughing_train.mining.requests; + +public record MineNextProject(String analyzerName) {} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/requests/StoreResults.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/requests/StoreResults.java new file mode 100644 index 000000000..7759a4346 --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/requests/StoreResults.java @@ -0,0 +1,7 @@ +package io.github.martinwitt.laughing_train.mining.requests; + +import io.github.martinwitt.laughing_train.data.Project; +import io.github.martinwitt.laughing_train.data.result.CodeAnalyzerResult; +import java.io.Serializable; + +public record StoreResults(Project project, CodeAnalyzerResult result, String analyzerName) implements Serializable {} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectService.java index fc7b16e20..9cb36a27f 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectService.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectService.java @@ -1,18 +1,16 @@ package io.github.martinwitt.laughing_train.services; -import com.google.common.base.Strings; import com.google.common.flogger.FluentLogger; import io.github.martinwitt.laughing_train.data.Project; import io.github.martinwitt.laughing_train.data.ProjectRequest; import io.github.martinwitt.laughing_train.data.ProjectResult; -import io.smallrye.mutiny.Uni; -import io.smallrye.mutiny.unchecked.Unchecked; import io.vertx.core.Vertx; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.util.Optional; import java.util.Random; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -37,17 +35,14 @@ public ProjectResult handleProjectRequest(ProjectRequest request) { String repoName = StringUtils.substringAfterLast(url.url(), "/").replace(".git", ""); Path dir = Files.createTempDirectory("laughing-train-" + repoName + random.nextLong()); cleanAfter60min(dir); - return checkoutRepo(url, dir) - .onItem() - .invoke(() -> logger.atInfo().log("Cloned %s to %s", url.url(), dir)) - .onFailure() - .invoke(e -> logger.atSevere().withCause(e).log("Error while cloning %s to %s", url.url(), dir)) - .onFailure() - .invoke(e -> FileUtils.deleteQuietly(dir.toFile())) - .onItemOrFailure() - .transform((git, error) -> toResult(url, repoName, dir, git, error)) - .await() - .indefinitely(); + Optional checkoutRepo = checkoutRepo(url, dir); + if (checkoutRepo.isEmpty()) { + FileUtils.deleteQuietly(dir.toFile()); + return new ProjectResult.Error("Could not checkout repo"); + } + Git git = checkoutRepo.get(); + logger.atInfo().log("Cloned %s to %s", url.url(), dir); + return new ProjectResult.Success(new Project(repoName, url.url(), dir.toFile(), ".", getHash(git))); } } catch (Exception e) { logger.atSevere().withCause(e).log("Error while handling project request %s", request); @@ -64,16 +59,6 @@ private void cleanAfter60min(Path dir) { }); } - private ProjectResult toResult(ProjectRequest.WithUrl url, String repoName, Path dir, Git git, Throwable error) { - if (error == null) { - String commitHash = getHash(git); - return new ProjectResult.Success(new Project(repoName, url.url(), dir.toFile(), ".", commitHash)); - } else { - git.close(); - return new ProjectResult.Error(Strings.nullToEmpty(error.getMessage())); - } - } - private String getHash(Git git) { try { return ObjectId.toString(git.log().call().iterator().next().getId()); @@ -82,18 +67,21 @@ private String getHash(Git git) { } } - private Uni checkoutRepo(ProjectRequest.WithUrl url, Path dir) { + private Optional checkoutRepo(ProjectRequest.WithUrl url, Path dir) { return createAsyncRepo(url, dir); } - private Uni createAsyncRepo(ProjectRequest.WithUrl url, Path dir) { - return Uni.createFrom().item(Unchecked.supplier(() -> { + private Optional createAsyncRepo(ProjectRequest.WithUrl url, Path dir) { + try { FileUtils.deleteDirectory(dir.toFile()); Files.createDirectories(dir); - return Git.cloneRepository() + return Optional.ofNullable(Git.cloneRepository() .setURI(url.url()) .setDirectory(dir.toFile()) - .call(); - })); + .call()); + } catch (Exception e) { + logger.atSevere().withCause(e).log("Error while cloning %s to %s", url.url(), dir); + return Optional.empty(); + } } } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 033e24c4c..000000000 Binary files a/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 9f4197d5f..000000000 --- a/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,7 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip -networkTimeout=10000 -validateDistributionUrl=true -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/settings.gradle b/settings.gradle index 55b242992..2ced83c14 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,2 +1,5 @@ +plugins { + id("org.gradle.toolchains.foojay-resolver-convention") version "0.6.0" +} rootProject.name = 'laughing-train-project' -include(':code-transformation',":commons", ":github-bot", ":application", ":matcher", ":spoon-analyzer") \ No newline at end of file +include(':code-transformation',":commons", ":github-bot", ":application", ":matcher", ":spoon-analyzer")