Problem to get into the next page, Selenium

Question

I'm new in using Selenium, I wanted scrap 2Gis it is analogue of GoogleMaps but I faced the problem with the clicking to the next page. What is the problem in the provided code, especially why selenium can not find file path button to get to the next page?

This is my provided code

import requests
from bs4 import BeautifulSoup
import csv
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import csv
import time
from opencage.geocoder import OpenCageGeocode
import time
from selenium.common.exceptions import TimeoutException




key = "......"
geocoder = OpenCageGeocode(key)



def get_data():
    headers = {
        'User-Agent':,
    }
    
    s = Service('/usr/local/bin/chromedriver')
    driver = webdriver.Chrome(service=s)
    driver.get(f"https://2gis.kz/almaty/search/vape%20shop/page/1")
    

    with open('vape_data.csv', 'w', encoding="utf-8") as csvfile:
        csv_writer = csv.writer(csvfile)
        csv_writer.writerow(['Name', 'Location', "Longitude", "Latitude"])
        
        
        while True:
            time.sleep(2)  # Wait for page to load
        
            page_source = driver.page_source
            soup = BeautifulSoup(page_source, "lxml")
            supermarkets = soup.find_all("div", class_="_1kf6gff")
        
            for item in supermarkets:
                name = item.find("span", class_="_1al0wlf").text.strip()
                distc = item.find("span", class_="_1w9o2igt").text.strip()
                latitude, longitude = get_coordinates(distc, geocoder)
            
                print(f"{name} and {distc}")
            
                with open("vape_data.csv", "a", encoding="utf-8") as file:
                    csv_writer = csv.writer(file)
                    csv_writer.writerow([name, distc, longitude, latitude])
                
                time.sleep(0.5)  # Respectful scraping pause
        
        # Try to find and click the Next button
            try:
                next_button = driver.find_element(By.XPATH, 'x_path')  # Update this XPATH
                next_button.click()
            except Exception as e:
                print("No more pages or an error occurred.", str(e))
                break  # Exit the loop if Next button not found or error occurs
    
    print("Finished scraping.")
    driver.quit()  # Close the browser

def main():
    get_data()
 
    
if __name__ == "__main__":
    main()

It scraps shops in the first page and shutdowns, printing:

print("No more pages or an error occurred.", str(e))

Link to the map: https://2gis.kz/almaty/search/vape%20shop

Also, you can ignore part of the related to the GeoCoding.

Answer 1

Here a solution for next page XPATH

I changed find_element to find_elements and get last one [-1]

driver.find_elements(By.XPATH, "//div[./div/a[contains(@href, 'page')]]//*[local-name() = 'svg']")[-1]