Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update parallelcurl.php #25

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Update parallelcurl.php #25

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 24 additions & 32 deletions parallelcurl.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
<?php
//ParallelCurl with bug fixes for PHP 7.x
//This file is forked from https://github.com/petewarden/ParallelCurl


// This class is designed to make it easy to run multiple curl requests in parallel, rather than
// waiting for each one to finish before starting the next. Under the hood it uses curl_multi_exec
Expand Down Expand Up @@ -32,12 +35,9 @@
// associated with this object. This cookie contains user-defined data.
//
// By Pete Warden <[email protected]>, freely reusable, see http://petewarden.typepad.com for more

class ParallelCurl {

public $max_requests;
public $options;

public $outstanding_requests;
public $multi_handle;

Expand All @@ -53,7 +53,6 @@ public function __construct($in_max_requests = 10, $in_options = array()) {
public function __destruct() {
$this->finishAllRequests();
}

// Sets how many requests can be outstanding at once before we block and wait for one to
// finish before starting the next one
public function setMaxRequests($in_max_requests) {
Expand All @@ -62,23 +61,19 @@ public function setMaxRequests($in_max_requests) {

// Sets the options to pass to curl, using the format of curl_setopt_array()
public function setOptions($in_options) {

$this->options = $in_options;
}

// Start a fetch from the $url address, calling the $callback function passing the optional
// $user_data value. The callback should accept 3 arguments, the url, curl handle and user
// data, eg on_request_done($url, $ch, $user_data);
public function startRequest($url, $callback, $user_data = array(), $post_fields=null) {

if( $this->max_requests > 0 )
$this->waitForOutstandingRequestsToDropBelow($this->max_requests);
if( $this->max_requests > 0 )
$this->waitForOutstandingRequestsToDropBelow($this->max_requests);

$ch = curl_init();
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt_array($ch, $this->options);
curl_setopt($ch, CURLOPT_URL, $url);

if (isset($post_fields)) {
curl_setopt($ch, CURLOPT_POST, TRUE);
curl_setopt($ch, CURLOPT_POSTFIELDS, $post_fields);
Expand All @@ -87,7 +82,6 @@ public function startRequest($url, $callback, $user_data = array(), $post_fields
curl_multi_add_handle($this->multi_handle, $ch);

$ch_array_key = (int)$ch;

$this->outstanding_requests[$ch_array_key] = array(
'url' => $url,
'callback' => $callback,
Expand All @@ -102,8 +96,8 @@ public function startRequest($url, $callback, $user_data = array(), $post_fields
public function finishAllRequests() {
$this->waitForOutstandingRequestsToDropBelow(1);
}

// Checks to see if any of the outstanding requests have finished

private function checkForCompletedRequests() {
/*
// Call select to see if anything is waiting for us
Expand All @@ -119,15 +113,15 @@ private function checkForCompletedRequests() {
do {
$mrc = curl_multi_exec($this->multi_handle, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);

while ($active && $mrc == CURLM_OK) {
if (curl_multi_select($this->multi_handle) != -1) {
if (curl_multi_select($this->multi_handle, 0.01) >0) {
do {
$mrc = curl_multi_exec($this->multi_handle, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
else
}else{
return;
}
}

// Now grab the information about the completed requests
Expand All @@ -140,36 +134,34 @@ private function checkForCompletedRequests() {
die("Error - handle wasn't found in requests: '$ch' in ".
print_r($this->outstanding_requests, true));
}

$request = $this->outstanding_requests[$ch_array_key];

$url = $request['url'];
$content = curl_multi_getcontent($ch);
$callback = $request['callback'];
$user_data = $request['user_data'];

call_user_func($callback, $content, $url, $ch, $user_data);

unset($this->outstanding_requests[$ch_array_key]);

curl_multi_remove_handle($this->multi_handle, $ch);
curl_multi_remove_handle($this->multi_handle, $ch);
call_user_func($callback, $content, $url, $ch, $user_data);

}

}

// Blocks until there's less than the specified number of requests outstanding
private function waitForOutstandingRequestsToDropBelow($max)
{
private function waitForOutstandingRequestsToDropBelow($max){
$usleep = 0;
while (1) {
$this->checkForCompletedRequests();
if (count($this->outstanding_requests)<$max)
if (count($this->outstanding_requests)<$max){
break;

usleep(10000);
}else{
$usleep++;
usleep(100000);
if($usleep >= 300){
//continue after 30s, to prevent hung requests from causing an infinite loop
break;
}
}
}
}

}


?>