132 lines
4.5 KiB
Python
132 lines
4.5 KiB
Python
# file: tool/aiqicha_detail.py
|
||
|
||
import time
|
||
import json
|
||
from tool.web_browser import WebBrowser
|
||
from tool.aiqicha_login import AiqichaLoginManager # 导入登录管理器
|
||
from tool.aiqicha_detail_parser import AiqichaDetailParser # 导入解析器
|
||
|
||
class AiqichaDetailCrawler:
|
||
def __init__(self, cookie_path="aiqicha_cookies.json"):
|
||
self.browser = WebBrowser(cookie_path)
|
||
self.browser_started = False
|
||
self.login_manager = None # 添加登录管理器实例
|
||
|
||
|
||
def start_browser(self):
|
||
"""启动浏览器"""
|
||
if not self.browser_started:
|
||
try:
|
||
self.browser.start_browser()
|
||
# 初始化登录管理器
|
||
self.login_manager = AiqichaLoginManager(self.browser)
|
||
|
||
# 加载cookies
|
||
if not self.browser.load_cookies():
|
||
print("未找到有效Cookie")
|
||
else:
|
||
print("已加载Cookie")
|
||
|
||
# 使用登录管理器检测登录状态
|
||
# logined = self.login_manager.check_and_login()
|
||
# if logined:
|
||
# print("登录成功")
|
||
# else:
|
||
# print("登录失败")
|
||
self.browser_started = True
|
||
except Exception as e:
|
||
print(f"启动浏览器失败: {e}")
|
||
self.browser_started = False
|
||
|
||
def close_browser(self):
|
||
"""关闭浏览器"""
|
||
if self.browser_started:
|
||
try:
|
||
# 保存cookies
|
||
self.browser.save_cookies()
|
||
self.browser.close_browser()
|
||
except Exception as e:
|
||
print(f"关闭浏览器时出错: {e}")
|
||
finally:
|
||
self.browser_started = False
|
||
|
||
def crawl_company_detail(self, url: str, refer_url: str = None):
|
||
"""
|
||
爬取爱企查企业详情页数据
|
||
|
||
Args:
|
||
url (str): 企业详情页URL，例如 https://aiqicha.baidu.com/company_detail_45719927199916
|
||
|
||
Returns:
|
||
dict: 包含企业详细信息的字典
|
||
"""
|
||
if not self.browser_started:
|
||
self.start_browser()
|
||
|
||
if not self.browser_started:
|
||
print("浏览器未启动，无法执行爬取")
|
||
return {}
|
||
|
||
print(f'正在爬取企业详情: {url}')
|
||
|
||
try:
|
||
# 设置 Referer 头部模拟搜索引擎点击
|
||
if refer_url:
|
||
self.browser.page.set_extra_http_headers({"Referer": refer_url})
|
||
|
||
# 访问页面
|
||
if self.browser.visit_page(url):
|
||
# 增强页面加载检查
|
||
print("等待页面关键元素加载...")
|
||
try:
|
||
# 等待关键元素加载，增加超时时间
|
||
self.browser.page.wait_for_selector('.addr-enter-bg-ele', timeout=15000)
|
||
print("关键元素已加载")
|
||
|
||
# 额外等待一段时间确保页面完全加载
|
||
import time
|
||
time.sleep(10)
|
||
print("额外等待完成，页面应该已完全加载")
|
||
except Exception as e:
|
||
print(f"等待页面元素时出错: {e}")
|
||
print("继续尝试解析页面内容...")
|
||
|
||
|
||
self.browser.save_cookies()
|
||
|
||
# 提取基本信息
|
||
print("开始解析页面信息...")
|
||
parser = AiqichaDetailParser(self.browser)
|
||
company_info = parser.parse_company_info()
|
||
|
||
print(f"成功爬取企业信息: {company_info['name']}")
|
||
return company_info
|
||
else:
|
||
print("访问页面失败")
|
||
return {}
|
||
except Exception as e:
|
||
print(f"爬取过程中出现错误: {e}")
|
||
return {}
|
||
|
||
|
||
def __enter__(self):
|
||
"""上下文管理器入口"""
|
||
self.start_browser()
|
||
return self
|
||
|
||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||
"""上下文管理器出口"""
|
||
self.close_browser()
|
||
|
||
|
||
# 使用示例:
|
||
# 方法1: 手动管理浏览器生命周期
|
||
# crawler = QiqichaDetailCrawler()
|
||
# crawler.start_browser()
|
||
# detail = crawler.crawl_company_detail("https://aiqicha.baidu.com/company_detail_45719927199916")
|
||
# crawler.close_browser()
|
||
|
||
# 方法2: 使用上下文管理器
|
||
# with QiqichaDetailCrawler() as crawler:
|
||
# detail = crawler.crawl_company_detail("https://aiqicha.baidu.com/company_detail_45719927199916")
|
||
# print(detail) |