Skip to content

Commit

Permalink
fix(vertx): Run Spoon and Qodana mining in sync (#822)
Browse files Browse the repository at this point in the history
  • Loading branch information
@MartinWitt
MartinWitt authored Jul 10, 2023
1 parent 26dcf19 commit 46529d2
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 72 deletions.
15 changes: 7 additions & 8 deletions github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/QodanaPeriodicMiner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ public class QodanaPeriodicMiner {
final QodanaService qodanaService;
final ProjectService projectService;
MeterRegistry registry;
final SpoonPeriodicMiner spoonPeriodicMiner;

private final Random random = new Random();
private Queue<Project> queue = new ArrayDeque<>();
Expand All @@ -54,14 +55,16 @@ public QodanaPeriodicMiner(
ProjectRepository projectRepository,
QodanaService qodanaService,
ProjectService projectService,
MiningPrinter miningPrinter) {
MiningPrinter miningPrinter,
SpoonPeriodicMiner spoonPeriodicMiner) {
this.registry = registry;
this.vertx = vertx;
this.searchProjectService = searchProjectService;
this.projectRepository = projectRepository;
this.qodanaService = qodanaService;
this.projectService = projectService;
this.miningPrinter = miningPrinter;
this.spoonPeriodicMiner = spoonPeriodicMiner;
}

private Project getRandomProject() throws IOException {
Expand All @@ -81,9 +84,7 @@ void mine(@Observes StartupEvent event) {
try {
logger.atInfo().log("Starting Qodana periodic miner");
vertx.exceptionHandler(it -> logger.atWarning().withCause(it).log("Exception in vertx"));
vertx.setTimer(TimeUnit.MINUTES.toMillis(3), v -> vertx.createSharedWorkerExecutor(
"MINING", 5, 30L, TimeUnit.MINUTES)
.executeBlocking(it -> mineRandomRepo()));
vertx.setTimer(TimeUnit.MINUTES.toMillis(3), v -> vertx.executeBlocking(it -> mineRandomRepo()));
} catch (Exception e) {
logger.atWarning().withCause(e).log("Failed to repo with Qodana");
}
Expand All @@ -101,6 +102,7 @@ private void mineRandomRepo() {
}
if (checkoutResult instanceof ProjectResult.Success success) {
String commitHash = success.project().commitHash();
spoonPeriodicMiner.mineRandomRepo(success);
if (isAlreadyMined(success, commitHash, ANALYZER_NAME)) {
logger.atInfo().log(
"Project %s already analyzed with commit hash %s", success.project(), commitHash);
Expand All @@ -125,11 +127,8 @@ private void mineRandomRepo() {
logger.atWarning().withCause(e).log("Failed to mine random repo");
registry.counter("mining.error").increment();
} finally {
logger.atInfo().log("Queue size: %s", queue.size());
logger.atInfo().log("Mining next repo in 1 minute");
vertx.setTimer(TimeUnit.MINUTES.toMillis(1), v -> vertx.createSharedWorkerExecutor(
"MINING", 5, 30L, TimeUnit.MINUTES)
.executeBlocking(it -> mineRandomRepo()));
vertx.setTimer(TimeUnit.MINUTES.toMillis(1), v -> vertx.executeBlocking(it -> mineRandomRepo()));
}
}

Expand Down
76 changes: 12 additions & 64 deletions github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/SpoonPeriodicMiner.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package io.github.martinwitt.laughing_train.mining;

import com.google.common.flogger.FluentLogger;
import io.github.martinwitt.laughing_train.data.ProjectRequest;
import io.github.martinwitt.laughing_train.data.ProjectResult;
import io.github.martinwitt.laughing_train.data.ProjectResult.Success;
import io.github.martinwitt.laughing_train.data.request.AnalyzerRequest;
Expand All @@ -14,16 +13,12 @@
import io.github.martinwitt.laughing_train.services.SpoonAnalyzerService;
import io.micrometer.core.instrument.MeterRegistry;
import io.quarkus.arc.Unremovable;
import io.quarkus.runtime.StartupEvent;
import io.vertx.core.Vertx;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Observes;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Queue;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;

@Unremovable
Expand All @@ -39,7 +34,6 @@ public class SpoonPeriodicMiner {
final ProjectService projectService;
MeterRegistry registry;
final SpoonAnalyzerService spoonAnalyzerService;
private final Random random = new Random();
private Queue<Project> queue = new ArrayDeque<>();

public SpoonPeriodicMiner(
Expand All @@ -59,69 +53,27 @@ public SpoonPeriodicMiner(
this.spoonAnalyzerService = spoonAnalyzerService;
}

private Project getRandomProject() throws IOException {
if (random.nextBoolean()) {
return searchProjectService.searchProjectOnGithub();
} else {
return getKnownProject();
}
}

private Project getKnownProject() {
var list = projectRepository.getAll();
return list.get(random.nextInt(list.size()));
}

void mine(@Observes StartupEvent event) {
void mineRandomRepo(ProjectResult.Success success) {
try {
logger.atInfo().log("Start mining with spoon");
vertx.exceptionHandler(it -> logger.atWarning().withCause(it).log("Exception in vertx"));
vertx.setTimer(TimeUnit.MINUTES.toMillis(3), v -> vertx.createSharedWorkerExecutor(
"MINING", 5, 30L, TimeUnit.MINUTES)
.executeBlocking(it -> mineRandomRepo()));
} catch (Exception e) {
logger.atWarning().withCause(e).log("Failed to repo with spoon");
}
}

private void mineRandomRepo() {
try {
logger.atInfo().log("Start mining with spoon");
var project = queue.isEmpty() ? getRandomProject() : queue.poll();
var checkoutResult = checkoutProject(project);
if (checkoutResult instanceof ProjectResult.Error) {
logger.atWarning().log("Failed to checkout project %s", project);
mineRandomRepo();
return;
String commitHash = success.project().commitHash();
if (isAlreadyMined(success, commitHash, ANALYZER_NAME)) {
logger.atInfo().log("Project %s already analyzed with commit hash %s", success.project(), commitHash);
}
if (checkoutResult instanceof ProjectResult.Success success) {
String commitHash = success.project().commitHash();
if (isAlreadyMined(success, commitHash, ANALYZER_NAME)) {
logger.atInfo().log(
"Project %s already analyzed with commit hash %s", success.project(), commitHash);
tryDeleteProject(success);
mineRandomRepo();
}
logger.atInfo().log("Successfully checked out project %s", success.project());
var spoonResult = analyzeProjectWithSpoon(success);
if (spoonResult instanceof CodeAnalyzerResult.Failure error) {
logger.atWarning().log("Failed to analyze project with spoon %s", error.message());
tryDeleteProject(success);
}
if (spoonResult instanceof CodeAnalyzerResult.Success successResult) {
addOrUpdateCommitHash(success, spoonResult);
}
logger.atInfo().log("Successfully checked out project %s", success.project());
var spoonResult = analyzeProjectWithSpoon(success);
if (spoonResult instanceof CodeAnalyzerResult.Failure error) {
logger.atWarning().log("Failed to analyze project with spoon %s", error.message());
tryDeleteProject(success);
}
if (spoonResult instanceof CodeAnalyzerResult.Success successResult) {
addOrUpdateCommitHash(success, spoonResult);
}
} catch (Exception e) {
logger.atWarning().withCause(e).log("Failed to mine random repo");
registry.counter("mining.error").increment();
} finally {
logger.atInfo().log("Queue size: %s", queue.size());
logger.atInfo().log("Mining next repo in 1 minute");
vertx.setTimer(TimeUnit.MINUTES.toMillis(1), v -> vertx.createSharedWorkerExecutor(
"MINING", 5, 30L, TimeUnit.MINUTES)
.executeBlocking(it -> mineRandomRepo()));
logger.atInfo().log("Finished mining with spoon");
}
}

Expand All @@ -140,10 +92,6 @@ private boolean isAlreadyMined(ProjectResult.Success success, String commitHash,
.anyMatch(v -> v.getAnalyzerName().equals(analyzerName));
}

private ProjectResult checkoutProject(Project project) throws IOException {
return projectService.handleProjectRequest(new ProjectRequest.WithUrl(project.getProjectUrl()));
}

private void tryDeleteProject(ProjectResult.Success project) {
try {
FileUtils.deleteDirectory(project.project().folder());
Expand Down

0 comments on commit 46529d2

Please sign in to comment.