# 定义大模型
from langchain_openai import ChatOpenAI
llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo-0613")
# 定义提取方法
def extract(content: str, schema: dict):
from langchain.chains import create_extraction_chain
return create_extraction_chain(schema=schema, llm=llm).invoke(content)
import pprint
from langchain_text_splitters import RecursiveCharacterTextSplitter
def scrape_with_playwright(urls, schema):
# 加载数据
loader = AsyncChromiumLoader(urls)
docs = loader.load()
# 数据转换
bs_transformer = BeautifulSoupTransformer()
# 提取其中的span标签
docs_transformed = bs_transformer.transform_documents(
docs, tags_to_extract=["span"]
)
# 数据切分
splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
chunk_size=1000, chunk_overlap=0)
splits = splitter.split_documents(docs_transformed)
# 因为数据量太大,输入第一片数据使用,传入使用的架构
extracted_content = extract(schema=schema, content=splits[0].page_content)
pprint.pprint(extracted_content)
return extracted_content
urls = ["https://ceshiren.com/"]
schema = {
"properties": {
"title": {"type": "string"},
"url": {"type": "string"},
},
"required": ["title", "url"],
}
extracted_content = scrape_with_playwright(urls, schema=schema)
评论