From 702bd9eec1827485977c4f97b906f4ae73f016a7 Mon Sep 17 00:00:00 2001 From: Mike Griffith Date: Thu, 9 Jan 2014 16:11:06 +0000 Subject: [PATCH 1/2] Added support for custom HTTP headers per curl handle. Misc. reformatting. --- parallelcurl.php | 93 ++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/parallelcurl.php b/parallelcurl.php index 28fb508..0a3686e 100755 --- a/parallelcurl.php +++ b/parallelcurl.php @@ -28,7 +28,7 @@ // // The callback should take four arguments. The first is a string containing the content found at // the URL. The second is the original URL requested, the third is the curl handle of the request that -// can be queried to get the results, and the fourth is the arbitrary 'cookie' value that you +// can be queried to get the results, and the fourth is the arbitrary 'cookie' value that you // associated with this object. This cookie contains user-defined data. // // By Pete Warden , freely reusable, see http://petewarden.typepad.com for more @@ -37,21 +37,19 @@ class ParallelCurl { public $max_requests; public $options; - public $outstanding_requests; public $multi_handle; - + public function __construct($in_max_requests = 10, $in_options = array()) { $this->max_requests = $in_max_requests; $this->options = $in_options; - $this->outstanding_requests = array(); $this->multi_handle = curl_multi_init(); } - - //Ensure all the requests finish nicely + + // Ensure all the requests finish nicely public function __destruct() { - $this->finishAllRequests(); + $this->finishAllRequests(); } // Sets how many requests can be outstanding at once before we block and wait for one to @@ -59,7 +57,7 @@ public function __destruct() { public function setMaxRequests($in_max_requests) { $this->max_requests = $in_max_requests; } - + // Sets the options to pass to curl, using the format of curl_setopt_array() public function setOptions($in_options) { @@ -69,23 +67,27 @@ public function setOptions($in_options) { // Start a fetch from the $url address, calling the $callback function passing the optional // $user_data value. The callback should accept 3 arguments, the url, curl handle and user // data, eg on_request_done($url, $ch, $user_data); - public function startRequest($url, $callback, $user_data = array(), $post_fields=null) { + public function startRequest($url, $callback, $user_data = array(), $post_fields = null, $headers = null) { + if( $this->max_requests > 0 ) { + $this->waitForOutstandingRequestsToDropBelow($this->max_requests); + } - if( $this->max_requests > 0 ) - $this->waitForOutstandingRequestsToDropBelow($this->max_requests); - $ch = curl_init(); curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); - curl_setopt_array($ch, $this->options); curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt_array($ch, $this->options); if (isset($post_fields)) { curl_setopt($ch, CURLOPT_POST, TRUE); curl_setopt($ch, CURLOPT_POSTFIELDS, $post_fields); } - + + if (isset($headers)) { + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + } + curl_multi_add_handle($this->multi_handle, $ch); - + $ch_array_key = (int)$ch; $this->outstanding_requests[$ch_array_key] = array( @@ -93,10 +95,10 @@ public function startRequest($url, $callback, $user_data = array(), $post_fields 'callback' => $callback, 'user_data' => $user_data, ); - + $this->checkForCompletedRequests(); } - + // You *MUST* call this function at the end of your script. It waits for any running requests // to complete, and calls their callback functions public function finishAllRequests() { @@ -105,71 +107,68 @@ public function finishAllRequests() { // Checks to see if any of the outstanding requests have finished private function checkForCompletedRequests() { - /* + /* // Call select to see if anything is waiting for us if (curl_multi_select($this->multi_handle, 0.0) === -1) return; - // Since something's waiting, give curl a chance to process it do { $mrc = curl_multi_exec($this->multi_handle, $active); } while ($mrc == CURLM_CALL_MULTI_PERFORM); */ + // fix for https://bugs.php.net/bug.php?id=63411 - do { - $mrc = curl_multi_exec($this->multi_handle, $active); - } while ($mrc == CURLM_CALL_MULTI_PERFORM); - - while ($active && $mrc == CURLM_OK) { - if (curl_multi_select($this->multi_handle) != -1) { - do { - $mrc = curl_multi_exec($this->multi_handle, $active); - } while ($mrc == CURLM_CALL_MULTI_PERFORM); - } - else - return; - } - + do { + $mrc = curl_multi_exec($this->multi_handle, $active); + } while ($mrc == CURLM_CALL_MULTI_PERFORM); + + while ($active && $mrc == CURLM_OK) { + if (curl_multi_select($this->multi_handle) != -1) { + do { + $mrc = curl_multi_exec($this->multi_handle, $active); + } while ($mrc == CURLM_CALL_MULTI_PERFORM); + } else { + return; + } + } + // Now grab the information about the completed requests while ($info = curl_multi_info_read($this->multi_handle)) { - + $ch = $info['handle']; $ch_array_key = (int)$ch; - + if (!isset($this->outstanding_requests[$ch_array_key])) { die("Error - handle wasn't found in requests: '$ch' in ". print_r($this->outstanding_requests, true)); } - + $request = $this->outstanding_requests[$ch_array_key]; $url = $request['url']; $content = curl_multi_getcontent($ch); $callback = $request['callback']; $user_data = $request['user_data']; - + call_user_func($callback, $content, $url, $ch, $user_data); - + unset($this->outstanding_requests[$ch_array_key]); - + curl_multi_remove_handle($this->multi_handle, $ch); } - + } - + // Blocks until there's less than the specified number of requests outstanding - private function waitForOutstandingRequestsToDropBelow($max) - { + private function waitForOutstandingRequestsToDropBelow($max) { while (1) { $this->checkForCompletedRequests(); if (count($this->outstanding_requests)<$max) - break; - + break; + usleep(10000); } } - } - ?> From 17d4b307035c4b5ad098399e01879e6ce31f966f Mon Sep 17 00:00:00 2001 From: Mike Griffith Date: Thu, 9 Jan 2014 16:16:15 +0000 Subject: [PATCH 2/2] whitespace --- test.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test.php b/test.php index 57009fc..edc4f4c 100755 --- a/test.php +++ b/test.php @@ -1,8 +1,8 @@ #!/usr/bin/php