package main
import (
"net/http"
"io"
"strconv"
"fmt"
"os"
)
func get(url string) (re string,e error) {
resp,err:=http.Get(url)
if err!=nil {
e=err
return
}
defer resp.Body.Close()
buf:=make([]byte,4096)
for {
n,err:=resp.Body.Read(buf)
re+=string(buf[:n])
if err!=nil {
if err==io.EOF {
break
} else {
e=err
return
}
}
}
return
}
func spider(start int,end int) {
m:=make(chan int)
for i:=start; i<=end; i++ {
go func(i int) {
re,err:=get("https://tieba.baidu.com/f?kw=%E7%BB%9D%E5%9C%B0%E6%B1%82%E7%94%9F&ie=utf-8&pn="+strconv.Itoa(50*(i-1)))
if err!=nil {
fmt.Print(err)
return
}
fp,err:=os.Create("/Users/zmx/"+strconv.Itoa(i)+".txt")
if err!=nil {
fmt.Print(err)
return
}
defer fp.Close()
fp.WriteString(re)
m<-i
}(i)
}
for i:=start; i<=end; i++ {
<-m
}
}
func main() {
var start int
fmt.Scan(&start)
var end int
fmt.Scan(&end)
spider(start,end)
}
go语言实现百度贴吧并发爬虫
猜你喜欢
转载自blog.csdn.net/baidu_25845567/article/details/82386706
今日推荐
周排行