import requests
import lxml
from bs4 import BeautifulSoup
import csv
import os

with open('res.csv', 'w', newline='', encoding='utf-8-sig') as file:
    writer = csv.writer(file, delimiter=';')
    writer.writerow(["№", "category", "section", "name", "price", "stock", "link"])

url = 'https://inshi.by/katalog'
schema = 'https://inshi.by'
fake_ua = {'user-agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)'}
links = []
items = []

response = requests.get(url=url, headers=fake_ua)
response.encoding = 'utf-8'

soup = BeautifulSoup(response.text, 'lxml')

cats_lists = soup.find_all('ul', class_='side-menu')

for i in cats_lists[0].find_all('li')[1:]:
    links.append(schema + i.find('a').get('href'))
for i in cats_lists[1].find_all('li')[1:]:
    links.append(schema + i.find('a').get('href'))

print('got cats')

for link in links:
    response1 = requests.get(url=link, headers=fake_ua)
    response1.encoding = 'utf-8'
    soup1 = BeautifulSoup(response1.text, 'lxml')
    href = soup1.find_all('div', class_='col-md-4 col-sm-12 col-xs-12 product-wrapper js-product')
    for i in href:
        items.append(schema + '/' + i.find('span', 'prod-title').find('a').get('href'))

print('got items')

os.mkdir("docs")
for i, item in enumerate(items):
    try:
        response2 = requests.get(url=item, headers=fake_ua)
        response2.encoding = 'utf-8'
        soup2 = BeautifulSoup(response2.text, 'lxml')
        category = soup2.find('ol', class_='breadcrumb').find_all('li')[3].text.strip()
        section = soup2.find('ol', class_='breadcrumb').find_all('li')[4].text.strip()
        name = soup2.find('div', class_='col-md-12 text-center').find('h1').text.strip()
        price = ''
        os.mkdir(f'docs/{i + 1} {name}')
        try:
            price = soup2.find('div', class_='single-product-price').find('p', class_='normal-price').text.strip()
        except Exception as q:
            print(q)
        stock = soup2.find('div', class_='sin-product-add-cart').findNext('p').text.strip()
        try:
            doc1 = schema + '/' + soup2.find('a', class_='js-leave-testi btn btn-primary').find_previous_sibling('p').\
                find('a').get('href')
            filedata = requests.get(doc1)
            with open(f"docs/{i+1} {name}/{name}", 'wb') as f:
                f.write(filedata.content)
        except Exception as r:
            print(r)
        with open('res.csv', 'a', newline='', encoding='utf-8-sig') as file:
            writer = csv.writer(file, delimiter=';')
            writer.writerow([i + 1, category, section, name, price, stock, item])
    except Exception as e:
        print(e, item)
    print(i+1)