分享我经常用的 php多线程源码 批量抓取内容的方法
<?phpclass Http_MultiRequest
{
//要并行抓取的url 列表
private $urls = array();
//curl 的选项
private $options;
//构造函数
function __construct($options = array())
{
$this->setOptions($options);
}
//设置url 列表
function setUrls($urls)
{
$this->urls = $urls;
return $this;
}
//设置选项
function setOptions($options)
{
$options = 1;
if (isset($options['HTTP_POST']))
{
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $options['HTTP_POST']);
unset($options['HTTP_POST']);
}
if (!isset($options))
{
$options = 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1;)';
}
if (!isset($options))
{
$options = 1;
}
if (!isset($options))
{
$options = 0;
}
$this->options = $options;
}
//并行抓取所有的内容
function exec()
{
if(empty($this->urls) || !is_array($this->urls))
{
return false;
}
$curl = $data = array();
$mh = curl_multi_init();
foreach($this->urls as $k => $v)
{
$curl[$k] = $this->addHandle($mh, $v);
}
$this->execMulitHandle($mh);
foreach($this->urls as $k => $v)
{
$data[$k] = curl_multi_getcontent($curl[$k]);
curl_multi_remove_handle($mh, $curl[$k]);
}
curl_multi_close($mh);
return $data;
}
//只抓取一个网页的内容。
function execOne($url)
{
if (empty($url)) {
return false;
}
$ch = curl_init($url);
$this->setOneOption($ch);
$content = curl_exec($ch);
curl_close($ch);
return $content;
}
//内部函数，设置某个handle 的选项
private function setOneOption($ch)
{
curl_setopt_array($ch, $this->options);
}
//添加一个新的并行抓取 handle
private function addHandle($mh, $url)
{
$ch = curl_init($url);
$this->setOneOption($ch);
curl_multi_add_handle($mh, $ch);
return $ch;
}
//并行执行(这样的写法是一个常见的错误，我这里还是采用这样的写法，这个写法
//下载一个小文件都可能导致cup占用100%, 并且，这个循环会运行10万次以上
//这是一个典型的不懂原理产生的错误。这个错误在PHP官方的文档上都相当的常见。）
private function execMulitHandle2($mh)
{
$running = null;
do {
curl_multi_exec($mh, $running);
} while ($running > 0);
}
//应该用这样的写法
private function execMulitHandle($mh){
$i = 0;
do {$mrc = curl_multi_exec($mh,$active); $i++;} while ($mrc == CURLM_CALL_MULTI_PERFORM);
while ($active && $mrc == CURLM_OK){
if (curl_multi_select($mh) != -1){
do {$mrc = curl_multi_exec($mh, $active); $i++;} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
$i++;
}
//var_dump($i);
}
}
/*下面是上面的类的一个测试的例子：*/
$urls = array("http://baidu.com", "http://baidu.com", "http://baidu.com", "http://baidu.com", "http://baidu.com", "http://baidu.com", "http://www.google.com", "http://www.sina.com.cn", );
$m = new Http_MultiRequest();
$t = microtime(true);
$m->setUrls($urls);
//parallel fetch（并行抓取）:
$data = $m->exec();
$parallel_time = microtime(true) - $t;
echo $parallel_time . "\n";
$t = microtime(true);
//serial fetch（串行抓取）:
foreach ($urls as $url)
{
$data[] = $m->execOne($url);
}
$serial_time = microtime(true) - $t;
echo $serial_time . "\n";
?> 多线程差不多都是用curl
有个rollingurl是非block模式的 河小马 发表于 2013-4-14 04:00 static/image/common/back.gif
多线程差不多都是用curl
因为最需要用到多线程的就是网络访问，等待响应有时候是个漫长的过程。。。 http://code.google.com/p/rolling-curl/source/checkout
rolling curl的源码地址，要svn checkout下来的 不错哦，用来升级一下我的代码，之前自己写得很简单的 luguo 发表于 2013-4-14 08:58 static/image/common/back.gif
有个rollingurl是非block模式的
Rolling cURL 并发机制真的挺不错啊thanks luguo 光大 发表于 2013-4-15 20:28 static/image/common/back.gif
Rolling cURL 并发机制真的挺不错啊thanks luguo
其实就多了一个callback呗 :lol 学习下，，， 对这方面认识不够http://6.movie2046.com/img/images/smile7.gifhttp://6.movie2046.com/img/images/smile8.gifhttp://6.movie2046.com/img/images/smile9.gif
http://6.movie2046.com/img/images/smile10.gifhttp://6.movie2046.com/img/images/smile11.gifhttp://6.movie2046.com/img/images/smile12.gifhttp://6.movie2046.com/img/images/smile13.gifhttp://6.movie2046.com/img/images/smile14.gifhttp://6.movie2046.com/img/images/smile15.gifhttp://6.movie2046.com/img/images/smile16.gifhttp://6.movie2046.com/img/images/smile17.gifhttp://6.movie2046.com/img/images/smile18.gifhttp://6.movie2046.com/img/images/smile19.gifhttp://6.movie2046.com/img/images/smile20.gifhttp://6.movie2046.com/img/images/smile21.gifhttp://6.movie2046.com/img/images/smile22.gif 增长了见识了呵呵呵 还是不太习惯
以前没用过curl的multi
内容少点，确实可以用这个，东西多了我还是倾向于消息队列，更稳定些 php网络操作效率太差，换个别的语言实现吧
学习了:P:P:P pthreads 也非常不错
页:
[1]