Linked Questions

Popular Questions

Not getting full json object from Scrapy response

Asked by At

I am new to scraping. Learning it to collect data for deep learning projects. So, I made a spider to scrape restaurant information from airbnb website. Spider:

import scrapy
from scrapy.exceptions import CloseSpider
import json


class AirbnbspiderSpider(scrapy.Spider):
    name = 'airbnbSpider'
    allowed_domains = ['https://www.airbnb.ie']

    def start_requests(self):
        yield scrapy.Request(url="https://www.airbnb.ie/api/v2/explore_tabs?version=1.4.5&satori_version=1.1.0&_format=for_explore_search_web&experiences_per_grid=20&items_per_grid=18&guidebooks_per_grid=20&auto_ib=false&fetch_filters=true&has_zero_guest_treatment=true&is_guided_search=true&is_new_cards_experiment=true&luxury_pre_launch=true&query_understanding_enabled=true&show_groupings=true&supports_for_you_v3=true&timezone_offset=330&client_session_id=079d2e2d-d31b-4412-a07d-1c0ae23d4e15&metadata_only=false&is_standard_search=true&tab_id=restaurant_tab&section_offset=2&items_offset=18&recommendation_item_cursor=&refinement_paths%5B%5D=%2Frestaurants&click_referer=t%3ASEE_ALL%7Csid%3A8154ef6e-7dd1-40e5-8226-b11874a96c77%7Cst%3ALOCATION_SUGGESTION_RESTAURANTS&map_toggle=false&federated_search_session_id=bb29efb6-e881-4997-9ed8-ca00fff421f8&screen_size=medium&query=New%20York&_intents=p1&key=d306zoyjsyarp7ifhu67rjxn52tv0t20&currency=EUR&locale=en-IE", callback=self.parse_Id)


    def parse_Id(self, response):
        data = json.loads(response.body.decode('utf-8'))
        restaurants = data.get('explore_tabs')[0].get('sections')[0].get('recommendation_items')

        if restaurants is None:
            raise CloseSpider('No restaurant is available in this city')

        for restaurant in restaurants:    
            yield scrapy.Request(url='https://www.airbnb.ie/api/v2/similar_restaurants/{0}?key=d306zoyjsyarp7ifhu67rjxn52tv0t20&currency=EUR&locale=en-IE&number_of_guests=2&_format=default'.format(restaurant.get('id')), callback=self.parse)


    def parse(self, response):
        yield {'yes':"No"}

Ignore last parse method.

Now the data I want to scrape is in sections list of this Json object. But when I run the spider. sections is returned as empty list in response( check bold letters).

Here's the object from response:

{'pagination_metadata': {'has_next_page': False}, 'tab_name': 'restaurants', 'sections': [], 'restaurant_tab_metadata': {'has_map': True, 'point_of_interests': [], 'filters': {'eligible_for_left_filters_panel': False, 'more_filters_counts': {'small': 0, 'default': 0}, 'more_filters_button': {'bg_color': '#008489', 'text': 'See restaurants', 'verified': False}, 'show_left_filters_panel': False, 'sections': [{'selected': False, 'filter_bar_title': 'Dates', 'title': 'Dates', 'subsections': [], 'collapse_state': 'expanded', 'experiments_metadata': [], 'items': [{'selected': False, 'type': 'date_picker', 'title': 'Dates', 'params': [{'key': 'checkin', 'value_type': 'string', 'delete': False, 'invisible_to_user': False}, {'key': 'checkout', 'value_type': 'string', 'delete': False, 'invisible_to_user': False}]}], 'bar_items': [], 'filter_section_id': 'date_picker'}, {'selected': False, 'filter_bar_title': 'Guests', 'title': 'Guests', 'subsections': [], 'collapse_state': 'expanded', 'experiments_metadata': [], 'items': [{'selected': False, 'type': 'stepper', 'title': 'Adults', 'params': [{'value_type': 'integer', 'key': 'adults', 'invisible_to_user': False, 'delete': False, 'value': '0'}], 'key': 'adults', 'metadata': {'min_value': 0, 'max_value': 16}, 'value': 0}, {'selected': False, 'type': 'stepper', 'title': 'Children', 'subtitle': 'Ages 2–12', 'params': [{'value_type': 'integer', 'key': 'children', 'invisible_to_user': False, 'delete': False, 'value': '0'}], 'key': 'children', 'metadata': {'min_value': 0, 'max_value': 5}, 'value': 0}, {'selected': False, 'type': 'stepper', 'title': 'Infants', 'subtitle': 'Under 2', 'params': [{'value_type': 'integer', 'key': 'infants', 'invisible_to_user': False, 'delete': False, 'value': '0'}], 'key': 'infants', 'metadata': {'min_value': 0, 'max_value': 5}, 'value': 0}], 'bar_items': [], 'filter_section_id': 'guest_picker'}, {'selected': False, 'filter_bar_title': 'Time of day', 'title': 'Time of day', 'subsections': [], 'experiments_metadata': [], 'items': [{'selected': False, 'type': 'checkbox', 'title': 'Breakfast', 'states': [], 'sub_type': 'RESTAURANT_SERVICE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_service_types', 'invisible_to_user': False, 'delete': False, 'value': 5}]}, {'selected': False, 'type': 'checkbox', 'title': 'Brunch', 'states': [], 'sub_type': 'RESTAURANT_SERVICE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_service_types', 'invisible_to_user': False, 'delete': False, 'value': 1}]}, {'selected': False, 'type': 'checkbox', 'title': 'Lunch', 'states': [], 'sub_type': 'RESTAURANT_SERVICE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_service_types', 'invisible_to_user': False, 'delete': False, 'value': 3}]}, {'selected': False, 'type': 'checkbox', 'title': 'Dinner', 'states': [], 'sub_type': 'RESTAURANT_SERVICE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_service_types', 'invisible_to_user': False, 'delete': False, 'value': 2}]}], 'bar_items': [], 'filter_section_id': 'restaurant_service_types', 'collapse_selected_subtitles': [], 'collapse_subtitle': 'Breakfast, Brunch, Lunch, +1 more'}, {'selected': False, 'filter_bar_title': 'Cuisines', 'title': 'Cuisines', 'subsections': [], 'experiments_metadata': [], 'items': [{'selected': False, 'type': 'checkbox', 'title': 'American', 'states': [], 'sub_type': 'RESTAURANT_CUISINE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_cuisine_types', 'invisible_to_user': False, 'delete': False, 'value': 'American'}]}, {'selected': False, 'type': 'checkbox', 'title': 'Italian', 'states': [], 'sub_type': 'RESTAURANT_CUISINE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_cuisine_types', 'invisible_to_user': False, 'delete': False, 'value': 'Italian'}]}, {'selected': False, 'type': 'checkbox', 'title': 'New American', 'states': [], 'sub_type': 'RESTAURANT_CUISINE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_cuisine_types', 'invisible_to_user': False, 'delete': False, 'value': 'New American'}]}, {'selected': False, 'type': 'checkbox', 'title': 'Seafood', 'states': [], 'sub_type': 'RESTAURANT_CUISINE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_cuisine_types', 'invisible_to_user': False, 'delete': False, 'value': 'Seafood'}]}, {'selected': False, 'type': 'checkbox', 'title': 'Mexican', 'states': [], 'sub_type': 'RESTAURANT_CUISINE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_cuisine_types', 'invisible_to_user': False, 'delete': False, 'value': 'Mexican'}]}, {'selected': False, 'type': 'checkbox', 'title': 'Japanese', 'states': [], 'sub_type': 'RESTAURANT_CUISINE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_cuisine_types', 'invisible_to_user': False, 'delete': False, 'value': 'Japanese'}]}, {'selected': False, 'type': 'checkbox', 'title': 'French', 'states': [], 'sub_type': 'RESTAURANT_CUISINE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_cuisine_types', 'invisible_to_user': False, 'delete': False, 'value': 'French'}]}, {'selected': False, 'type': 'checkbox', 'title': 'Mediterranean', 'states': [], 'sub_type': 'RESTAURANT_CUISINE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_cuisine_types', 'invisible_to_user': False, 'delete': False, 'value': 'Mediterranean'}]}, {'selected': False, 'type': 'checkbox', 'title': 'Indian', 'states': [], 'sub_type': 'RESTAURANT_CUISINE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_cuisine_types', 'invisible_to_user': False, 'delete': False, 'value': 'Indian'}]}, {'selected': False, 'type': 'checkbox', 'title': 'Pizza', 'states': [], 'sub_type': 'RESTAURANT_CUISINE_TYPE', 'params': [{'value_type': 'array', 'key': 'restaurant_cuisine_types', 'invisible_to_user': False, 'delete': False, 'value': 'Pizza'}]}], 'bar_items': [], 'filter_section_id': 'restaurant_cuisine_types', 'collapse_selected_subtitles': [], 'collapse_subtitle': 'American, Italian, +8 more'}], 'more_filters_ordering': {'small': ['restaurant_service_types', 'restaurant_cuisine_types'], 'default': []}, 'filter_bar_counts': {'small': [0, 0], 'default': [0, 0, 0, 0]}, 'filter_bar_ordering': {'small': ['date_picker', 'guest_picker'], 'default': ['date_picker', 'guest_picker', 'restaurant_service_types', 'restaurant_cuisine_types']}}}, 'tab_id': 'restaurant_tab', 'experiments_metadata': []}

Why this is happening?

Related Questions