我们使用phantomjs抓取页面,可能需要登录,但是又想要登录一次,以后每次都要输入密码,就需要设置cookie。
phantomjs设置cookie方法:
以ChromeDriver为例:
package com.tomtop.big.data.crawler.ebay; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.util.Date; import java.util.Set; import java.util.StringTokenizer; import java.util.concurrent.TimeUnit; import org.junit.Test; import org.openqa.selenium.By; import org.openqa.selenium.By.ByTagName; import org.openqa.selenium.Cookie; import org.openqa.selenium.JavascriptExecutor; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.chrome.ChromeDriver; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.client.RestTemplate; import com.tomtop.big.data.base.utils.TomtopDateUtil; import com.tomtop.big.data.crawler.browser.amazon.AmazonHtmlunitBrowser; import com.tomtop.big.data.crawler.proxy.amazon.ReviewForeignProxyHelp; //@RunWith(SpringJUnit4ClassRunner.class) //@ContextConfiguration(locations = { "classpath*:etc/spring/*" }) @SuppressWarnings("all") public class Phantomjstest { @Autowired protected ReviewForeignProxyHelp foreignProxyHelp; private AmazonHtmlunitBrowser htmlunitBrowser; @Autowired private RestTemplate template; Set<Cookie> set; @Test public void f1() throws Exception { while(true) { phantomjsTest(); Thread.sleep(3000); } } // @Test public void phantomjsTest() throws Exception { System.setProperty("webdriver.chrome.driver", "D:\\phantomjs\\phantomjs-2.1.1-windows\\bin\\chromedriver.exe"); ChromeDriver driver = new ChromeDriver(); driver.get("https://www.ebay.com/sh/ovw"); if(set != null) { for (Cookie cookie : set) { driver.manage().addCookie(cookie); } } driver.get("https://www.ebay.com/sh/ovw"); driver.manage().timeouts().implicitlyWait(1, TimeUnit.SECONDS); String currentUrl = driver.getCurrentUrl(); if (currentUrl.contains("signin")) { System.err.println("进入登陆页面"); // 登陆页面 WebElement useridEle = driver.findElementById("userid"); WebElement passEle = driver.findElementById("pass"); WebElement btnEle = driver.findElementById("sgnBt"); useridEle.sendKeys("tomtop_game"); passEle.sendKeys("runningMen*2012"); btnEle.submit(); set = driver.manage().getCookies(); for (Cookie cookie : set) { System.err.println(cookie.getDomain()); System.err.println(cookie.getName()); System.err.println(cookie.getPath()); System.err.println(cookie.getValue()); if(cookie.getExpiry() != null) { System.err.println(TomtopDateUtil.getFormat2().format(cookie.getExpiry())); } System.err.println("================================="); } } Thread.sleep(1000L); String windowHandle = driver.getWindowHandle(); driver.switchTo().window(windowHandle); WebElement positiveEle = driver .findElementByClassName("feedback-positive"); By by = new ByTagName("a"); WebElement element = positiveEle.findElement(by); System.err.println(element.getText()); driver.close(); driver.quit(); } }
关键的地方在于:你要请求两次,先请求一次,再设置cookie,然后再请求一次。
driver.get("https://www.ebay.com/sh/ovw");
if(set != null) {
for (Cookie cookie : set) {
driver.manage().addCookie(cookie);
}
}
driver.get("https://www.ebay.com/sh/ovw");