LCI/defi-rendu-legifrance
1
0
Fork 0
Code Issues Pull Requests Activity

Moved interpreter code into a function and ran flake8 (except for E501)

Browse Source
This commit is contained in:
Suwako Moriya 2018-11-11 09:39:34 +01:00
parent 6d215e2305
commit e709259a8f
2 changed files with 24 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
fetch.py
View File

@ -11,34 +11,34 @@ import re
# Variables # Variables
api = "https://master.apis.dev.openstreetmap.org/" # Testing, it should be replaced by https://api.openstreetmap.org/ when the program is finished api = "https://master.apis.dev.openstreetmap.org/" # Testing, it should be replaced by https://api.openstreetmap.org/ when the program is finished
def runfetch(url): def runfetch(url='https://www.legifrance.gouv.fr/eli/arrete/2018/10/12/PRMD1824595A/jo/texte'):
session=requests.Session() session = requests.Session()
req1=session.get(url) # Get the source code req1 = session.get(url) # Get the source code
text=req1.text.split('\n') text = req1.text.split('\n')
# Parse the HTML source code # Parse the HTML source code
text=['\n'.join(row.split('\n')[:row.split('\n').index("</tr>"):]) for row in '\n'.join(text[text.index('<br/>ANNEXES<br/>ANNEXE I</p>'):text.index('<div style="margin-top: 30px; margin-bottom:20px;" id="JORFSCTA000037493059" class="titreSection">Annexe </div>'):]).split('<tr>')[2::]] text = ['\n'.join(row.split('\n')[:row.split('\n').index("</tr>"):]) for row in '\n'.join(text[text.index('<br/>ANNEXES<br/>ANNEXE I</p>'):text.index('<div style="margin-top: 30px; margin-bottom:20px;" id="JORFSCTA000037493059" class="titreSection">Annexe </div>'):]).split('<tr>')[2::]]
diclist=[] diclist = []
for row in text: for row in text:
cols=[text.split('\n')[2][5::] if len(text.split('\n'))==3 else "" for text in row.split("</td")[::]] cols = [text.split('\n')[2][5::] if len(text.split('\n')) == 3 else "" for text in row.split("</td")[::]]
diclist.append({"id":cols[0],"commune":cols[1],"site":cols[2],"departement":cols[3],"zone":cols[4],"ministere":cols[5],"aerozone":cols[6]}) diclist.append({"id": cols[0], "commune": cols[1], "site": cols[2], "departement": cols[3], "zone": cols[4], "ministere": cols[5], "aerozone": cols[6]})
#dicformat:{id,commune,site,departement,zone,ministere,aerozone} # dicformat:{id,commune,site,departement,zone,ministere,aerozone}
return diclist return diclist
def dms2dd(dms): def dms2dd(dms):
coordslst = re.search("(\d{1,3})° (\d{2})' (\d{2},?\d{0,2})(?:\"|”|\'\')", dms).groups() coordslst = re.search("(\d{1,3})° (\d{2})' (\d{2},?\d{0,2})(?:\"|”|\'\')", dms).groups()
dd = float(coordslst[0].replace(",", ".")) + float(coordslst[1].replace(",", "."))/60 + float(coordslst[2].replace(",", "."))/3600 dd = float(coordslst[0].replace(",", ".")) + float(coordslst[1].replace(",", "."))/60 + float(coordslst[2].replace(",", "."))/3600
return dd return dd
if __name__ == '__main__':
diclist = runfetch('https://www.legifrance.gouv.fr/eli/arrete/2018/10/12/PRMD1824595A/jo/texte') # Fetch all the data from a french article def fetch(url='https://www.legifrance.gouv.fr/eli/arrete/2018/10/12/PRMD1824595A/jo/texte'):
url = 'https://www.legifrance.gouv.fr/eli/arrete/2018/10/12/PRMD1824595A/jo/texte' diclist = runfetch(url=url)
for index, zone in enumerate(diclist): for index, zone in enumerate(diclist):
for element in zone: for element in zone:
diclist[index][element] = re.sub("<br.{0,2}>", "\n", diclist[index][element]) # Replace <br>, <br /> and <br/> by \n diclist[index][element] = re.sub("<br.{0,2}>", "\n", diclist[index][element]) # Replace <br>, <br /> and <br/> by \n
# Make a the area # Make a the area
for area in diclist: for area in diclist:
@ -47,7 +47,7 @@ if __name__ == '__main__':
# Multiple polygons # Multiple polygons
if "polygones" in area["zone"]: if "polygones" in area["zone"]:
lst = [] lst = []
tmp = re.compile(".*zone.*").split(area["zone"]) # List of polygons tmp = re.compile(".*zone.*").split(area["zone"]) # List of polygons
del tmp[0] del tmp[0]
for polygon in tmp: for polygon in tmp:
lst.append(re.findall(".{0,35} ?:? ?.? ?: ?(\d{1,3}° \d{2}' \d{2},?\d{0,2}(?:\"|”) .) ?\/ ?(\d{1,3}° \d{2}' \d{2},?\d{0,2}(?:\"|”) .)", polygon)) lst.append(re.findall(".{0,35} ?:? ?.? ?: ?(\d{1,3}° \d{2}' \d{2},?\d{0,2}(?:\"|”) .) ?\/ ?(\d{1,3}° \d{2}' \d{2},?\d{0,2}(?:\"|”) .)", polygon))
@ -83,8 +83,12 @@ if __name__ == '__main__':
# Atolls # Atolls
elif area["zone"].strip() == "atolls et eaux territoriales incluses": elif area["zone"].strip() == "atolls et eaux territoriales incluses":
area["zone"] = [("cercle", "35KM", "138.9022", "21.82917"), ("cercle", "27KM", "138.7425", "22.23528A")] area["zone"] = [("cercle", "35KM", "138.9022", "21.82917"), ("cercle", "27KM", "138.7425", "22.23528A")]
return diclist
if __name__ == '__main__':
diclist = fetch() # Fetch all the data from a french article
# Print the dict (keep this code in the end of the file) # Print the dict (keep this code in the end of the file)
print("===== Dict =====") print("===== Dict =====")

4
fetchbs.py
View File

@ -1,10 +1,11 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
#-*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Imports # Imports
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
def runfetch(): def runfetch():
site = requests.get("https://www.legifrance.gouv.fr/eli/arrete/2018/10/12/PRMD1824595A/jo/texte").text site = requests.get("https://www.legifrance.gouv.fr/eli/arrete/2018/10/12/PRMD1824595A/jo/texte").text
soup = BeautifulSoup(site, "html.parser") soup = BeautifulSoup(site, "html.parser")
@ -16,5 +17,6 @@ def runfetch():
tds[-1].append(td.text) tds[-1].append(td.text)
print(trs) print(trs)
if __name__ == "__main__": if __name__ == "__main__":
runfetch() runfetch()