import requests
from bs4 import BeautifulSoup
from urllib.request import urlopen
from zipfile import ZipFile
from io import BytesIO
import sqlalchemy
from config import db_name
from config import db_user as user
from config import ip_server as ip
from config import pwd_postgres as pwd

url = 'https://data.riksdagen.se/dataset/katalog/dataset-dokument.html'

soup = BeautifulSoup(requests.get(url).content, 'lxml')

links = []
for link in soup.find_all('a', href=True):
    if 'json' in link['href']:
        links.append('https://data.riksdagen.se/' + link['href'])
    
for url in links:
    print(url)
    with urlopen(url) as zipresp:
        with ZipFile(BytesIO(zipresp.read())) as zfile:
            zfile.extractall('documents')