爬虫：财政部PPP项目库

# 功能说明：提取财政部PPP相关要素。# 先从各页提取项目链接，再按链接提取各项目要素，再进入下一页；“采购社会资本方式”从另一个链接提取# 日期在Excel里是字符串格式，需要转换成日期格式，粘贴到记事本或使用DATEVALUE函数import datetime,time,random,osimport requests,refrom bs4 import BeautifulSoupimport pandas as pdstarttime = datetime.datetime.now()startPage=":8086/pppcentral/map/getPPPList.do" #用于寻找每个项目的PROJ_IDbaseUrl=':8083/efmisweb/ppp/projectLibrary/getProjInfo.do?projId=' #与PROJ_ID拼接成各个项目的链接baseUrl2=':8083/efmisweb/ppp/projectLibrary/getProjInfoNational.do?projId=' #另外一个链接，提取“采购社会资本方式”数据myParams= {'queryPage': '1','projStateType':'0'} #翻页用，也可以加入其他选项myHeader={'Accept':'application/json, text/javascript, */*; q=0.01','Accept-Encoding':'gzip, deflate','Accept-Language':'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7','Connection':'keep-alive','Content-Type':'application/x-www-form-urlencoded; charset=UTF-8','Cookie':'JSESSIONID=7A442574FB143BC29453ECFA1A10CCC2; _site_id_cookie=82; clientlanguage=zh_CN; faspjsessionid=201xEsLqLL6_eaSNWmga7x05Z-tO19aSbqBEJZgBol-6t836nruw!1514000050; JSESSIONID=IF12fM4-OOkAeZCsfI-E3ErRwaXDf9AAuuANnExxdWHvMxHtrAo5!-549990464','Host':':8086','Origin':':8086','Referer':':8086/pppcentral/map/toPPPList.do','User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36','X-Requested-With':'XMLHttpRequest'}def gettotalPage():#得到总页数r=requests.get(startPage,params=myParams,headers=myHeader)regExp="\"totalPage\":\d*"regResult=re.findall(regExp,r.text)[0]totalPage=re.sub('\"totalPage\":',"",regResult)totalPage=eval(totalPage) #转化为数值print(totalPage)return totalPagedef gettotalCount():#得到项目总数r=requests.get(startPage,params=myParams,headers=myHeader)regExp="\"totalCount\":\d*"regResult=re.findall(regExp,r.text)[0]totalCount=re.sub('\"totalCount\":',"",regResult)totalCount=eval(totalCount) #转化为数值print(totalCount)return totalCountdef getPageurl(Page):#对每一页，得到所有项目ID、NAME，并生成相应链接PROJ_ID=[]PROJ_NAME=[]PROJ_URL=[]PROJ_URL2=[]myParams['queryPage']=str(Page)try:r=requests.get(startPage,params=myParams,headers=myHeader,timeout=30)except:r=requests.get(startPage,params=myParams,headers=myHeader,timeout=200)r.encoding = 'utf-8'regExp="PROJ_NAME.*?,"regResult=re.findall(regExp,r.text)for i in range(0,len(regResult)):regResult[i]=re.sub('PROJ_NAME\":\"','',regResult[i])regResult[i]=re.sub('\",','',regResult[i])PROJ_NAME.append(regResult[i].strip()) regExp="PROJ_RID.*?,"regResult=re.findall(regExp,r.text)for i in range(0,len(regResult)):regResult[i]=re.sub('PROJ_RID\":\"','',regResult[i])regResult[i]=re.sub('\",','',regResult[i])PROJ_ID.append(regResult[i]) PROJ_URL.append(baseUrl+regResult[i])PROJ_URL2.append(baseUrl2+regResult[i])urlList=list(zip(PROJ_ID,PROJ_NAME,PROJ_URL,PROJ_URL2))return urlListdef getData(projUrl1,projUrl2):#查找每个链接的数据dataDict={}try:html = requests.get(projUrl1,headers=myHeader,timeout=30)except:html = requests.get(projUrl1,headers=myHeader,timeout=100)html.encoding = 'utf-8'bsObj=BeautifulSoup(html.text,"lxml")cases = ["所在地区", "所属行业", "项目总投资", "所处阶段", "发起时间", "回报机制", "项目示范级别/批次", "合作期限", "运作方式", "采购社会资本方式"]for i in range(len(cases)-1):dataDict[cases[i]]=re.sub(r'[\s]','',bsObj.find("td",text=cases[i]).next_sibling.next_sibling.get_text()).strip() #空行也算一个siblingtry:html = requests.get(projUrl2,headers=myHeader,timeout=30)except:html = requests.get(projUrl2,headers=myHeader,timeout=100)html.encoding = 'utf-8'bsObj=BeautifulSoup(html.text,"lxml")dataDict[cases[-1]]=re.sub(r'[\s]','',bsObj.find("td",text=cases[-1]).next_sibling.next_sibling.get_text()).strip() #空行也算一个siblingtime.sleep(random.random()*0.5)return dataDicttotalPage=gettotalPage()totalCount=gettotalCount() #totalCount仅用于计算进度currentPage=0currentCount=0fileName='PPP_Database_'+str(time.strftime("%Y%m%d"))+'_Origin.xlsx'fileName2='PPP_Database_'+str(time.strftime("%Y%m%d"))+'_Final.xlsx'myHeaderName= ["项目名称","所在地区", "所属行业", "项目总投资（万元）", "所处阶段", "发起时间", "回报机制", "项目示范级别/批次", "合作期限（年）", "项目运作方式", "采购社会资本方式"]finalResult=[]#finalResult.append(myHeaderName)for i in range(1,totalPage+1):print('-------------------------------------------------------')currentPage=currentPage+1urlList=getPageurl(i)for k in range(0,len(urlList)):dataDict1={}dataDict1["项目名称"]=urlList[k][1]dataDict2=getData(urlList[k][2],urlList[k][3])finalResult.append(list(dataDict1.values())+list(dataDict2.values()))currentCount=currentCount+1print("进度：\t"+str(currentCount)+"/"+str(totalCount)+"行\t"+str(currentPage)+"/"+str(totalPage)+"页\t"+'%5.2f' %(100*currentCount/totalCount) + "%")tempResultOut=pd.DataFrame(columns=myHeaderName,data=finalResult)if(os.path.exists(fileName)):os.remove(fileName)writer=pd.ExcelWriter(fileName)tempResultOut.to_excel(writer,sheet_name='数据',index=False)writer.close()myHeaderName2= ["项目名称","所在地区1","所在地区2","所在地区3", "所属行业1","所属行业2", "项目总投资（万元）", "所处阶段", "发起时间", "回报机制", "项目示范级别/批次", "合作期限（年）", "项目运作方式", "采购社会资本方式"]finalResult2=[]for i in range(totalCount):tempList=[]tempList.append(finalResult[i][0])#append不能用赋值tempList=tempList+(finalResult[i][1].split("-->")+['',''])[0:3]tempList=tempList+(finalResult[i][2].split("->")+['',''])[0:2]tempList.append(eval(finalResult[i][3].replace("万元","").replace(",","")))tempList=tempList+finalResult[i][4:8]#tempList.append(eval(finalResult[i][8].replace("年","")))tempString=finalResult[i][8].replace("年","")if tempString=="":tempList.append(tempString)else:tempList.append(eval(tempString))tempList=tempList+finalResult[i][9:]finalResult2.append(tempList)tempResultOut=pd.DataFrame(columns=myHeaderName2,data=finalResult2)if(os.path.exists(fileName2)):os.remove(fileName2)writer=pd.ExcelWriter(fileName2)tempResultOut.to_excel(writer,sheet_name='数据',index=False)writer.close()print('-------------------------------------------------------')#计算程序运行时间endtime = datetime.datetime.now()def timeStr(s):if s<10:return '0'+str(s)else:return str(s)print("程序开始运行时间："+timeStr(starttime.hour)+":"+timeStr(starttime.minute)+":"+timeStr(starttime.second))print("程序结束运行时间："+timeStr(endtime.hour)+":"+timeStr(endtime.minute)+":"+timeStr(endtime.second))runTime=(endtime-starttime).secondsrunTimehour=runTime//3600 #除法并向下取整，整除runTimeminute=(runTime-runTimehour*3600)//60runTimesecond=runTime-runTimehour*3600-runTimeminute*60print("程序运行耗时："+str(runTimehour)+"时"+str(runTimeminute)+"分"+str(runTimesecond)+"秒")