defi-rendu-legifrance/fetch.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Nov  5 20:55:36 2018

@author: suwako
"""

import requests
import re


def runfetch(url):
    session=requests.Session()
    req1=session.get(url) # Get the source code
    text=req1.text.split('\n')
    # Parse the HTML source code
    text=['\n'.join(row.split('\n')[:row.split('\n').index("</tr>"):]) for row in '\n'.join(text[text.index('<br/>ANNEXES<br/>ANNEXE I</p>'):text.index('<div style="margin-top: 30px; margin-bottom:20px;" id="JORFSCTA000037493059" class="titreSection">Annexe </div>'):]).split('<tr>')[2::]]
    diclist=[]
    for row in text:
        cols=[text.split('\n')[2][5::] if len(text.split('\n'))==3 else "" for text in row.split("</td")[::]]
        diclist.append({"id":cols[0],"commune":cols[1],"site":cols[2],"departement":cols[3],"zone":cols[4],"ministere":cols[5],"aerozone":cols[6]})
        #dicformat:{id,commune,site,departement,zone,ministere,aerozone}
    return diclist


if __name__ == '__main__':
    diclist=runfetch('https://www.legifrance.gouv.fr/eli/arrete/2018/10/12/PRMD1824595A/jo/texte') # Fetch all the data from a french article
    url='https://www.legifrance.gouv.fr/eli/arrete/2018/10/12/PRMD1824595A/jo/texte'
    for index, zone in enumerate(diclist):
        for element in zone:
            diclist[index][element] = re.sub("<br.{0,2}>", "\n", diclist[index][element]) # Replace <br>, <br /> and <br/> by \n
    # Make a the area
    for area in diclist:
        # Polygons
        if "polygone" in area["zone"] or "Polygone" in area["zone"]:
            lst = re.findall(".{0,35} ?:? ?.? ?: ?(\d{1,3}° \d{2}' \d{2},?\d{0,2}(?:\"|”) .) ?\/ ?(\d{1,3}° \d{2}' \d{2},?\d{0,2}(?:\"|”) .)", area["zone"])
            area["zone"] = lst
            del lst
        elif "cercle" in area["zone"] or "Cercle" in area["zone"]:
            lst = [()]
            lst[0] = re.search(r"(.{6}) .{0,5}?(\d{1,3},?\d{0,2} ?.{1,2}) .{1,35} ?(\d{3}° \d{1,2}' \d{1,2}” .) ?\/ ?(\d{1,3}° \d{1,2}' \d{1,2}” .)", area["zone"], re.S).groups()
            area["zone"] = lst
            del lst

    # Print the dict (keep this code in the end of the file)
    print("===== Dict =====")
    for index, area in enumerate(diclist):
        print(f"\n-------------{index}----------------\n")
        for element in area:
            print(f"    {element}: {diclist[index][element]}")
    print("\n")