Namularbre/crawler2.0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: The crawler no fetch text from span and div

Browse Source
This commit is contained in:
Namu
2025-11-16 01:49:55 +01:00
parent 37f75a90e5
commit c10df0ec95
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawler2/spiders/__init__.py
View File

@@ -19,7 +19,7 @@ class CrawlerSpider(scrapy.Spider):
yield { yield {
"url": response.url, "url": response.url,
"title": response.css("title::text").get(), "title": response.css("title::text").get(),
"content": " ".join(response.css("p, h1, h2, h3, h4, h5, a::text").getall()), "content": " ".join(response.css("p, h1, h2, h3, h4, h5, a::text, span::text, div::text").getall()),
"links": response.css("a::attr(href)").getall() "links": response.css("a::attr(href)").getall()
} }