→ clansoda: 请问C大，这种网页是不是完全爬不了 06/21 16:20
→ clansoda: http://www.family.com.tw/marketing/inquiry.aspx 06/21 16:21
→ clansoda: 外面能爬的那个网页，他大概有几百个缺漏值，可能他后来 06/21 16:21
→ clansoda: 就不曾更新了，因此有些新开的他就没加进去了 06/21 16:21
好读版：http://pastebin.com/zNjkb0uh
csv结果：http://www.mediafire.com/download/a3d7aat94yhz16n/familyMart_loc.csv
我用查邮递区号方式去对inquery.aspx查询
这里最重要的是用curl的referer
我一开始找不到httr的方法去设定，所以先用了RCurl去做
后面补上用httr的GET的做法
library(httr)
library(xml2)
library(pipeR)
library(purrr)
library(stringi)
library(stringr)
library(RCurl)
library(jsonlite)
url_postCodes <- "http://www.easytravel.com.tw/postid_search.asp"
all_postCodes <- url_postCodes %>>% GET %>>% content(encoding = "big5") %>>%
xml_find_all("//td/font") %>>% xml_text %>>%
stri_conv(from = "UTF-8", to = "Big5") %>>% str_extract("\\d{3}") %>>%
`[`(!is.na(.)) %>>% as.integer
# method with RCurl::getURL
familyMartLoc <- lapply(all_postCodes, function(postCode){
url_toPrint <- str_c("http://api.map.com.tw/net/GraphicsXY.aspx?",
"search_class=Zip&Zip=%i&fun=getCityByZipReturn")
city_area <- sprintf(url_toPrint, postCode) %>>%
getURL(referer = "http://www.family.com.tw/marketing/inquiry.aspx") %>>%
str_replace_all("getCityByZipReturn\\(|\\)", "") %>>% fromJSON
url_toPrint <- str_c("http://api.map.com.tw/net/familyShop.aspx?",
"searchType=ShopList&type=&city=%s&area=%s&road=&",
"fun=showStoreList&key=6F30E8BF706D653965BDE302661D1241F8BE9EBC")
sprintf(url_toPrint, city_area$COUNTY, city_area$TOWN) %>>%
getURL(referer = "http://www.family.com.tw/marketing/inquiry.aspx") %>>%
str_replace_all("showStoreList\\(|\\)", "") %>>% fromJSON
}) %>>% do.call(what = rbind)
# method with httr::GET
familyMartLoc2 <- lapply(all_postCodes, function(postCode){
url_toPrint <- str_c("http://api.map.com.tw/net/GraphicsXY.aspx?",
"search_class=Zip&Zip=%i&fun=getCityByZipReturn")
city_area <- sprintf(url_toPrint, postCode) %>>%
GET(config(referer = "http://www.family.com.tw/marketing/inquiry.aspx")) %>>%
content("text") %>>% str_replace_all("getCityByZipReturn\\(|\\)", "") %>>%
fromJSON
url_toPrint <- str_c("http://api.map.com.tw/net/familyShop.aspx?",
"searchType=ShopList&type=&city=%s&area=%s&road=&",
"fun=showStoreList&key=6F30E8BF706D653965BDE302661D1241F8BE9EBC")
sprintf(url_toPrint, city_area$COUNTY, city_area$TOWN) %>>%
GET(config(referer = "http://www.family.com.tw/marketing/inquiry.aspx")) %>>%
content("text") %>>% str_replace_all("showStoreList\\(|\\)", "") %>>%
fromJSON
}) %>>% do.call(what = rbind)
partial results:
NAME TEL POSTel px py
1 全家新德店 02-23214351 02-77463163 121.5316 25.04447
2 全家台医店 02-23278841 02-23279603 121.5185 25.04079
3 全家南机场店 02-23039420 02-77095865 121.5058 25.02840
4 全家新惠安店 02-23036887 02-77300717 121.5065 25.02953
5 全家新龙口店 02-23047184 02-77300715 121.5104 25.02723
6 全家新厦门店 02-23681443 02-77397215 121.5176 25.02532