爬取私募排排网历史净值和破解加密数值(完整版)

from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
import pandas
import time
import re
from lxml import html
from selenium.webdriver.common.action_chains import ActionChains        # 导入鼠标事件库

from selenium import webdriver
driver = webdriver.Chrome()         # 启动驱动器
driver.get('https://www.simuwang.com/user/option')       # 加载网站

driver = webdriver.Chrome()         # 启动驱动器

# 谷歌浏览器 79和79版本后防止被检测
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
    "source": """
    Object.defineProperty(navigator, 'webdriver', {
      get: () => undefined
    })
  """
})
driver.get('https://www.simuwang.com/user/option')       # 加载网站

driver.find_element(By.XPATH,'//button[@class="comp-login-method comp-login-b2"]').click()      #点击账号密码登录
driver.find_element(By.XPATH,'//input[@name="username"]').send_keys('xxxxxxxxxxxx')      # 输入账号
driver.find_element(By.XPATH,'//input[@type="password"]').send_keys('xxxxxxxxxxxx')        # 输入密码
driver.find_element(By.XPATH,'//button[@style="margin-top: 65px;"]').click()            # 点击登录

time.sleep(15)              # 等待登录时间
driver.find_element(By.XPATH,'//span[@class="el-icon-close close-icon"]').click()       # 叉掉广告
driver.back()               # 网页后退

time.sleep(5)       # 加载网页需要等待时间
mouse = driver.find_element(By.XPATH,'//div[@class="comp-header-nav-item fz14"]/div/span[@class="ellipsis"]')
ActionChains(driver).move_to_element(mouse).perform()       # 悬停鼠标在名片
driver.find_element(By.XPATH,'//a[@class="comp-header-user-item icon-trade"]').click()      # 点击自选

# 解析网页
page = driver.page_source
soup = BeautifulSoup(page,'html.parser')

list_url = []   # 用于保存目标网站
list_name = []  # 用于保存目标名称
url_a = soup.select('div:nth-child(2) > div.shortName > a')    # 找到所爬取的网页
names = soup.select('div> div > div:nth-child(2) > div.shortName > a')  # 找到名称
for u in url_a:
    url = u['href']     # 得到网站
    list_url.append(url)
for name in names:
    list_name.append(name.get_text())

driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
    "source": """
    Object.defineProperty(navigator, 'webdriver', {
      get: () => undefined
    })
  """
})
for ind in range(len(list_name)):
    driver.get(list_url[ind])       # 加载网站

driver.find_element(By.XPATH,'//div/div[2]/div[2]/div[1]/div[2]/div[1]/div[1]/a[2]').click()        # 点击历史净值

for i in range(50):
    js = 'document.getElementsByClassName("tbody")[0].scrollTop=100000'  # 在历史净值中滑动，这里滑动50应该是都够了的，如果不够加大就行
    driver.execute_script(js)
    time.sleep(0.1)  # 防止滑动太快，没有读取到结果

page_url = driver.page_source  # 解析当前网页

.m440e0{font: 0/0 a;color: transparent;text-shadow: none;background-color: transparent;border: 0;}.m446e7{font: 0/0 a;color: transparent;text-shadow: none;background-color: transparent;border: 0;}.m48eb7{text-shadow: none;background-color: transparent;border: 0;}.m45029{text-shadow: none;background-color: transparent;border: 0;}.m41fd7{font: 0/0 a;color: transparent;text-shadow: none;background-color: transparent;border: 0;}.m4dec4{text-shadow: none;background-color: transparent;border: 0;}.m44109{text-shadow: none;background-color: transparent;border: 0;}

# 找到隐藏的属性值
def getHideIds(htmlEtree):
    encode_styles = "".join(htmlEtree.xpath('//div[@id="ENCODE_STYLE"]/style/text()')).replace("\n", "")
    # 清洗数据，去除连续的空格
    new_encode_styles = re.sub("  +", "", encode_styles)
    # 获取全部被隐藏的id
    hideIds1 = re.findall("\.(\w+) {font: 0/0 a;", new_encode_styles)  # 格式化后的html
    hideIds2 = re.findall("\.(\w+){font: 0/0 a;", new_encode_styles)  # 未格式化的html
    result = set(hideIds1 + hideIds2)
    return result

htmlEtree = etree.HTML(text=htmlData)

# 获取被隐藏的id
hideIds = getHideIds(htmlEtree)

# 处理数据
divList = htmlEtree.xpath('//div[@class="tr flex-h-center"]')
# print(divList)
tdDivs = []
for div in divList:
    nextDivs = div.xpath('./div[@class="td flex-h-center"]')
    for nextDiv in nextDivs:
        if nextDivs.index(nextDiv) == 0:
            continue
        tdDivs.append(nextDiv)

resultList = []
for tdDiv in tdDivs:

    labels = tdDiv.xpath("./*")
    nowResultList = []
    for label in labels:
        classStr = label.xpath("./@class")[0]
        if classStr not in hideIds:
            nowResultList.append(label.xpath("./text()")[0])
    resultList.append("".join(nowResultList))

# print(resultList)
# for reslut in resultList:
#     print(reslut)

write = pandas.ExcelWriter(r"C:\Users\86178\Desktop\私募排排网历史净值爬取.xlsx")   # 新建xlsx文件。
list_info.append([list_date[index], resultList[i], resultList[i + 1], resultList[i + 2], list_net[index]]) # 分别对应净值日期，单位净值，累计净值，累计净值，净值变动

pd = pandas.DataFrame(list_info, columns=['净值日期', '单位净值', '累计净值（分红再投资）', '累计净值（分红不投资）', '净值变动'])
        # print(pd)
pd.to_excel(write, sheet_name=list_name[ind], index=False)

write.save()  # 这里一定要保存