Feat: The crawler no fetch text from span and div

2025-11-16 01:49:55 +01:00
parent 37f75a90e5
commit c10df0ec95
1 changed files with 1 additions and 1 deletions
--- a/crawler2/spiders/init.py
+++ b/crawler2/spiders/init.py
@@ -19,7 +19,7 @@ class CrawlerSpider(scrapy.Spider):
        yield {
            "url": response.url,
            "title": response.css("title::text").get(),
-            "content": " ".join(response.css("p, h1, h2, h3, h4, h5, a::text").getall()),
+            "content": " ".join(response.css("p, h1, h2, h3, h4, h5, a::text, span::text, div::text").getall()),
            "links": response.css("a::attr(href)").getall()
        }