defi-rendu-legifrance/fetch.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Nov  5 20:55:36 2018

@author: suwako
"""

import requests
import re


def runfetch(url):
    session=requests.Session()
    req1=session.get(url) # Get the source code
    text=req1.text.split('\n')
    # Parse the HTML source code
    text=['\n'.join(row.split('\n')[:row.split('\n').index("</tr>"):]) for row in '\n'.join(text[text.index('<br/>ANNEXES<br/>ANNEXE I</p>'):text.index('<div style="margin-top: 30px; margin-bottom:20px;" id="JORFSCTA000037493059" class="titreSection">Annexe </div>'):]).split('<tr>')[2::]]
    diclist=[]
    for row in text:
        cols=[text.split('\n')[2][5::] if len(text.split('\n'))==3 else "" for text in row.split("</td")[::]]
        diclist.append({"id":cols[0],"commune":cols[1],"site":cols[2],"departement":cols[3],"zone":cols[4],"ministere":cols[5],"aerozone":cols[6]})
        #dicformat:{id,commune,site,departement,zone,ministere,aerozone}
    return diclist


if __name__ == '__main__':
    diclist=runfetch('https://www.legifrance.gouv.fr/eli/arrete/2018/10/12/PRMD1824595A/jo/texte') # Fetch all the data from a french article
    url='https://www.legifrance.gouv.fr/eli/arrete/2018/10/12/PRMD1824595A/jo/texte'
    for index, zone in enumerate(diclist):
        for element in zone:
            diclist[index][element] = re.sub("<br.{0,2}>", "\n", diclist[index][element]) # Replace <br>, <br /> and <br/> by \n
    # Print the dict
    print("===== Dict =====")
    for index, zone in enumerate(diclist):
        print(f"\n-------------{index}----------------\n")
        for element in zone:
            print(f"    {element}: {diclist[index][element]}")