import requests

#Root search URL parsing with BeautifulSoup
url = 'https://www.pararius.com/apartments/amsterdam/0-1500'
response = requests.get(url)

#Root page url and pages subdirectories
url = 'https://www.pararius.com/apartments/amsterdam/0-1500'#+page (for the other pages)
response = requests.get(url)

#BeautifulSoup Html parsing
html_soup = BeautifulSoup(response.text, 'html.parser')

#Individual ads are stored in the li tag, class search-list__item search-list__item--listing
#Here the line retrieves all containers (ads) of that type
add_containers = html_soup.find_all('li', class_ = 'search-list__item search-list__item--listing')

from time import sleep
import numpy as np
import pandas as pd
from random import uniform
from shapely.geometry import Point

#Preparation of list for each feature
addresses = []
prices = []
dict_features = [] #will host all the features
links = []
geoms = []

#loop trough containers corresponding to the n ads within a page
for container in add_containers:
    #The ad link
    link = 'https://www.pararius.com/'+container.find('a',class_ = 'listing-search-item__link listing-search-item__link--title',href=True)['href']
    links.append(link)

    # Pausing your execution with sleep is essential as to not overload the website.
    # We don't want to create any problems or been refused access.
    sleep(uniform(0.2, 0.5))
    
    # Access and parse the ad page html
    response2 = requests.get(link)
    html_soup2 = BeautifulSoup(response2.text, 'html.parser')

    # The address
    address1 = container.section.h2.a.contents[0]
    address2 = html_soup2.findAll('div', class_ = "listing-detail-summary__location")[0].contents[0].replace('n','').replace('        ','')
    addresses.append(address2)

    # Price
    price = float(html_soup2.findAll('meta', {'itemprop':'price'})[0]['content'])
    prices.append(price)

    # Loop through <dd> tags to get all the features
    features = {i.contents[0]:i.findNext("dd").string for i in  html_soup2.findAll('dt',class_ = 'listing-features__term')}
    dict_features.append(features)

    # Coordinates are stroed into a sapely Point object
    geom = Point([float(html_soup2.findAll('div', class_ = "detail-map map")[0]['data-detail-map-longitude']),
               float(html_soup2.findAll('div', class_ = "detail-map map")[0]['data-detail-map-latitude'])])
    geoms.append(geom)

#Create Dataframe for features lists    
ad_df = pd.DataFrame([addresses,prices,dict_features,links,geoms]).T
ad_df.columns = ['adress','Rental price 0','features','link','geometry']

# Create DataFrame from Series containing feature dictionaries
features_df = ad_df['features'].apply(pd.Series)
ad_df = pd.concat([ad_df.drop('features', axis=1), features_df],join='inner', axis=1)

import geopandas as gpd

#store result in GeoJSON file    
all_ad_gdf = gpd.GeoDataFrame(all_ad_df, geometry = 'geometry', crs="EPSG:4326")
all_ad_gdf.to_file("ad_Amsterdam.geojson", driver='GeoJSON')

import shapely

#Define polygon to filter our ads
filter_polygon_wkt = 'POLYGON((4.9580351217396235 52.349463547269636,4.950825343907592 52.340759453326854,4.91941131192517 52.33048012411839,4.905335079015014 52.331319342591414,4.884049068272827 52.3417033535761,4.8701444967396235 52.34096921067909,4.85627425728536 52.35397222330489,4.85352767525411 52.35806108990049,4.853012691123251 52.36193992133192,4.874985347373251 52.373259968432215,4.879963527304891 52.38441043375573,4.893696437461141 52.383781777115416,4.924252162558798 52.37634199381422,4.937985072715048 52.37424605391656,4.9580351217396235 52.349463547269636))'
filter_polygon = shapely.wkt.loads(filter_polygon_wkt)

filter_gdf['price/m2'] = filter_gdf['Rental price']/filter_gdf['Living area']

filter_gdf.sort_values('price/m2',inplace=True)

import osmnx as ox
import networkx as nx

def route_distance(point,point2,graph):
    #Retrieve the nearest node to the point of interest from the nodes geodataframe
    source = ox.distance.nearest_nodes(G, point.x,point.y)
    target = ox.distance.nearest_nodes(G,  point2.x,point2.y)
    
    #Compute the shortest path between the points of interest
    try:
        route_length = nx.shortest_path_length(graph, source=source, target=target,method='dijkstra',weight='length')
        return route_length
    except nx.NetworkXNoPath:
        return None

#Create multidigraph from the area of interest using OSMnx
G = ox.graph_from_polygon(filter_polygon,network_type='all_private')

#Create shapely points of the work place and train station coordinates
work = Point(4.913039, 52.342523)
station = Point(4.918278, 52.346691)