Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow queue jobs with batches #4127

Open
wants to merge 10 commits into
base: 3.1
Choose a base branch
from
15 changes: 13 additions & 2 deletions src/ChunkReader.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Jobs\SyncJob;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Bus;
use Maatwebsite\Excel\Concerns\ShouldBatch;
use Maatwebsite\Excel\Concerns\ShouldQueueWithoutChain;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithEvents;
Expand Down Expand Up @@ -37,7 +39,7 @@ public function __construct(Container $container)
* @param WithChunkReading $import
* @param Reader $reader
* @param TemporaryFile $temporaryFile
* @return PendingDispatch|Collection|null
* @return PendingDispatch|\Illuminate\Bus\PendingBatch|Collection|null
*/
public function read(WithChunkReading $import, Reader $reader, TemporaryFile $temporaryFile)
{
Expand Down Expand Up @@ -94,6 +96,14 @@ public function read(WithChunkReading $import, Reader $reader, TemporaryFile $te

$jobs->push($afterImportJob);

// Check if the import class is batchable
if ($import instanceof ShouldBatch) {
return Bus::batch([
$jobs->toArray(),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By doing this, it will execute as a chain with in the batch right? It won't run parallel?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's right. This creates a batch and returns a PendingBatch.

]);
}

// Check if the import class is queueable
if ($import instanceof ShouldQueue) {
return new PendingDispatch(
(new QueueImport($import))->chain($jobs->toArray())
Expand Down Expand Up @@ -133,7 +143,8 @@ protected function dispatchNow($command, $handler = null)
{
$uses = class_uses_recursive($command);

if (in_array(InteractsWithQueue::class, $uses) &&
if (
in_array(InteractsWithQueue::class, $uses) &&
in_array(Queueable::class, $uses) && !$command->job
) {
$command->setJob(new SyncJob($this->container, json_encode([]), 'sync', 'sync'));
Expand Down
25 changes: 25 additions & 0 deletions src/Concerns/BatchableTrait.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
<?php

namespace Maatwebsite\Excel\Concerns;

if (trait_exists('\Illuminate\Bus\Batchable')) {
trait BatchableTrait
{
use \Illuminate\Bus\Batchable;

public function batchCancelled()
{
$batch = $this->batch();

return $batch !== null && method_exists($batch, 'cancelled') && $batch->cancelled();
}
}
} else {
trait BatchableTrait
{
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can add the methods we need else where like batchCancelled here as well, so we don't have to do a method_exist check there. We can return false here

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

public function batchCancelled()
{
return false;
}
}
}
7 changes: 7 additions & 0 deletions src/Concerns/ShouldBatch.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<?php

namespace Maatwebsite\Excel\Concerns;

interface ShouldBatch
{
}
2 changes: 1 addition & 1 deletion src/Excel.php
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ public function import($import, $filePath, string $disk = null, string $readerTy
$readerType = FileTypeDetector::detect($filePath, $readerType);
$response = $this->reader->read($import, $filePath, $readerType, $disk);

if ($response instanceof PendingDispatch) {
if ($response instanceof PendingDispatch || $response instanceof \Illuminate\Bus\PendingBatch) {
return $response;
}

Expand Down
16 changes: 14 additions & 2 deletions src/Fakes/ExcelFake.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\PendingDispatch;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Queue;
use Illuminate\Support\Traits\Macroable;
use Maatwebsite\Excel\Concerns\ShouldBatch;
use Maatwebsite\Excel\Exporter;
use Maatwebsite\Excel\Importer;
use Maatwebsite\Excel\Reader;
Expand Down Expand Up @@ -102,6 +104,11 @@ public function handle()

Queue::push($this->job);

// Check if the export class is batchable
if ($export instanceof ShouldBatch) {
return Bus::batch([$this->job]);
}

return new PendingDispatch($this->job);
}

Expand All @@ -122,7 +129,7 @@ public function raw($export, string $writerType)
* @param string|UploadedFile $file
* @param string|null $disk
* @param string|null $readerType
* @return Reader|PendingDispatch
* @return Reader|PendingDispatch|\Illuminate\Bus\PendingBatch
*/
public function import($import, $file, string $disk = null, string $readerType = null)
{
Expand Down Expand Up @@ -174,7 +181,7 @@ public function toCollection($import, $file, string $disk = null, string $reader
* @param string|UploadedFile $file
* @param string|null $disk
* @param string $readerType
* @return PendingDispatch
* @return PendingDispatch|\Illuminate\Bus\PendingBatch
*/
public function queueImport(ShouldQueue $import, $file, string $disk = null, string $readerType = null)
{
Expand All @@ -197,6 +204,11 @@ public function handle()

Queue::push($this->job);

// Check if the import class is batchable
if ($import instanceof ShouldBatch) {
return Bus::batch([$this->job]);
}

return new PendingDispatch($this->job);
}

Expand Down
9 changes: 8 additions & 1 deletion src/Jobs/AfterImportJob.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@

use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\BatchableTrait;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Events\ImportFailed;
use Maatwebsite\Excel\HasEventBus;
Expand All @@ -14,7 +16,7 @@

class AfterImportJob implements ShouldQueue
{
use HasEventBus, InteractsWithQueue, Queueable;
use BatchableTrait, HasEventBus, InteractsWithQueue, Queueable, Dispatchable;

/**
* @var WithEvents
Expand Down Expand Up @@ -57,6 +59,11 @@ public function setDependencies(Collection $jobs)

public function handle()
{
// Determine if the batch has been cancelled...
if ($this->batchCancelled()) {
return;
}

foreach ($this->dependencyIds as $id) {
if (!ReadChunk::isComplete($id)) {
// Until there is no jobs left to run we put this job back into the queue every minute
Expand Down
8 changes: 7 additions & 1 deletion src/Jobs/AppendDataToSheet.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Maatwebsite\Excel\Concerns\BatchableTrait;
use Maatwebsite\Excel\Files\TemporaryFile;
use Maatwebsite\Excel\Jobs\Middleware\LocalizeJob;
use Maatwebsite\Excel\Writer;

class AppendDataToSheet implements ShouldQueue
{
use Queueable, Dispatchable, ProxyFailures, InteractsWithQueue;
use BatchableTrait, Queueable, Dispatchable, ProxyFailures, InteractsWithQueue;

/**
* @var array
Expand Down Expand Up @@ -73,6 +74,11 @@ public function middleware()
*/
public function handle(Writer $writer)
{
// Determine if the batch has been cancelled...
if ($this->batchCancelled()) {
return;
}

(new LocalizeJob($this->sheetExport))->handle($this, function () use ($writer) {
$writer = $writer->reopen($this->temporaryFile, $this->writerType);

Expand Down
8 changes: 7 additions & 1 deletion src/Jobs/AppendQueryToSheet.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Maatwebsite\Excel\Concerns\BatchableTrait;
use Maatwebsite\Excel\Concerns\FromQuery;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Events\AfterChunk;
Expand All @@ -16,7 +17,7 @@

class AppendQueryToSheet implements ShouldQueue
{
use Queueable, Dispatchable, ProxyFailures, InteractsWithQueue, HasEventBus;
use BatchableTrait, Queueable, Dispatchable, ProxyFailures, InteractsWithQueue, HasEventBus;

/**
* @var TemporaryFile
Expand Down Expand Up @@ -90,6 +91,11 @@ public function middleware()
*/
public function handle(Writer $writer)
{
// Determine if the batch has been cancelled...
if ($this->batchCancelled()) {
return;
}

(new LocalizeJob($this->sheetExport))->handle($this, function () use ($writer) {
if ($this->sheetExport instanceof WithEvents) {
$this->registerListeners($this->sheetExport->registerEvents());
Expand Down
8 changes: 7 additions & 1 deletion src/Jobs/AppendViewToSheet.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,15 @@
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Maatwebsite\Excel\Concerns\BatchableTrait;
use Maatwebsite\Excel\Concerns\FromView;
use Maatwebsite\Excel\Files\TemporaryFile;
use Maatwebsite\Excel\Jobs\Middleware\LocalizeJob;
use Maatwebsite\Excel\Writer;

class AppendViewToSheet implements ShouldQueue
{
use Queueable, Dispatchable, InteractsWithQueue;
use BatchableTrait, Queueable, Dispatchable, InteractsWithQueue;

/**
* @var TemporaryFile
Expand Down Expand Up @@ -68,6 +69,11 @@ public function middleware()
*/
public function handle(Writer $writer)
{
// Determine if the batch has been cancelled...
if ($this->batchCancelled()) {
return;
}

(new LocalizeJob($this->sheetExport))->handle($this, function () use ($writer) {
$writer = $writer->reopen($this->temporaryFile, $this->writerType);

Expand Down
10 changes: 9 additions & 1 deletion src/Jobs/CloseSheet.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,16 @@

use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Maatwebsite\Excel\Concerns\BatchableTrait;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Files\TemporaryFile;
use Maatwebsite\Excel\Writer;

class CloseSheet implements ShouldQueue
{
use Queueable, ProxyFailures;
use BatchableTrait, Queueable, Dispatchable, ProxyFailures, InteractsWithQueue;

/**
* @var object
Expand Down Expand Up @@ -54,6 +57,11 @@ public function __construct($sheetExport, TemporaryFile $temporaryFile, string $
*/
public function handle(Writer $writer)
{
// Determine if the batch has been cancelled...
if ($this->batchCancelled()) {
return;
}

$writer = $writer->reopen(
$this->temporaryFile,
$this->writerType
Expand Down
9 changes: 8 additions & 1 deletion src/Jobs/QueueExport.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Maatwebsite\Excel\Concerns\BatchableTrait;
use Maatwebsite\Excel\Concerns\WithMultipleSheets;
use Maatwebsite\Excel\Exceptions\NoSheetsFoundException;
use Maatwebsite\Excel\Files\TemporaryFile;
Expand All @@ -13,7 +15,7 @@

class QueueExport implements ShouldQueue
{
use ExtendedQueueable, Dispatchable;
use BatchableTrait, ExtendedQueueable, Dispatchable, InteractsWithQueue;

/**
* @var object
Expand Down Expand Up @@ -59,6 +61,11 @@ public function middleware()
*/
public function handle(Writer $writer)
{
// Determine if the batch has been cancelled...
if ($this->batchCancelled()) {
return;
}

(new LocalizeJob($this->export))->handle($this, function () use ($writer) {
$writer->open($this->export);

Expand Down
8 changes: 7 additions & 1 deletion src/Jobs/ReadChunk.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Support\Facades\Cache;
use Maatwebsite\Excel\Concerns\BatchableTrait;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithCustomValueBinder;
use Maatwebsite\Excel\Concerns\WithEvents;
Expand All @@ -24,7 +25,7 @@

class ReadChunk implements ShouldQueue
{
use Queueable, HasEventBus, InteractsWithQueue;
use BatchableTrait, Queueable, HasEventBus, InteractsWithQueue;

/**
* @var int
Expand Down Expand Up @@ -165,6 +166,11 @@ public function retryUntil()
*/
public function handle(TransactionHandler $transaction)
{
// Determine if the batch has been cancelled...
if ($this->batchCancelled()) {
return;
}

if (method_exists($this->import, 'setChunkOffset')) {
$this->import->setChunkOffset($this->startRow);
}
Expand Down
11 changes: 10 additions & 1 deletion src/Jobs/StoreQueuedExport.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,15 @@

use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Maatwebsite\Excel\Concerns\BatchableTrait;
use Maatwebsite\Excel\Files\Filesystem;
use Maatwebsite\Excel\Files\TemporaryFile;

class StoreQueuedExport implements ShouldQueue
{
use Queueable;
use BatchableTrait, Queueable, Dispatchable, InteractsWithQueue;

/**
* @var string
Expand All @@ -25,6 +28,7 @@ class StoreQueuedExport implements ShouldQueue
* @var TemporaryFile
*/
private $temporaryFile;

/**
* @var array|string
*/
Expand All @@ -49,6 +53,11 @@ public function __construct(TemporaryFile $temporaryFile, string $filePath, stri
*/
public function handle(Filesystem $filesystem)
{
// Determine if the batch has been cancelled...
if ($this->batchCancelled()) {
return;
}

$filesystem->disk($this->disk, $this->diskOptions)->copy(
$this->temporaryFile,
$this->filePath
Expand Down
Loading