fazziclay/VSTU_Schedule_Parser
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

work

Browse Source
This commit is contained in:
Станислав Миронов
2025-09-11 16:35:22 +03:00
parent babf491c8e
commit 6920d24a98
6 changed files with 173 additions and 276486 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -2,3 +2,5 @@
*.xlsx
__pycache__
.idea
result.json
groups.json

6149
groups.json
View File

File diff suppressed because it is too large Load Diff

53
links_parser.py Normal file
View File

@@ -0,0 +1,53 @@
import re
import time
from urllib.parse import urljoin
import requests
from requests.structures import CaseInsensitiveDict
from bs4 import BeautifulSoup
BASE_URL = "https://www.vstu.ru/"
RASP_PREFIX = "https://www.vstu.ru/student/raspisaniya/zanyatiy/index.php?dep="
def parse_links(facultets):
session = requests.Session()
session.headers = CaseInsensitiveDict(
{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
"Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"Referer": "http://dump.vstu.ru/",
"Upgrade-Insecure-Requests": "1",
"Priority": "u=0, i",
"Pragma": "no-cache",
"Cache-Control": "no-cach",
}
)
EXCEL_LINKS = {}
for facultet in facultets:
url = RASP_PREFIX + facultet
r = session.get(url)
print(f"GET {url}")
soup = BeautifulSoup(r.text, 'html.parser')
excel_pattern = re.compile(r'\.xlsx?$')
# Ищем все теги <a>, у которых атрибут href соответствует нашему паттерну
excel_tags = soup.find_all('a', href=excel_pattern)
excel_links = [tag.get('href') for tag in excel_tags]
# Предположим, вы уже получили excel_links из одного из методов выше
# excel_links = ['../../../upload/raspisanie/z/ОН_ХТФ_1 курс.xlsx', ...]
absolute_links = [urljoin(BASE_URL, relative_link) for relative_link in excel_links]
if facultet not in EXCEL_LINKS.keys():
EXCEL_LINKS[facultet] = set()
for excel_url in absolute_links:
EXCEL_LINKS[facultet].add(excel_url)
print(f"+url {excel_url}")
return EXCEL_LINKS

111
main.py
View File

@@ -8,7 +8,7 @@ import xlwt
import xlrd
import requests
from requests.structures import CaseInsensitiveDict
from bs4 import BeautifulSoup
import aigenerated
@@ -16,75 +16,28 @@ import parser
import translations
import utils
import json
import links_parser
# Общее правило проекта, сначала в координатах идёт ROW а потом COL, нумерация с нуля
FACULTETS = [
"asp", "mag", "fastiv", "fat", "ftkm", "ftpp", "feu", "fevt", "htf", "vkf", "mmf", "fpik"
]
BASE_URL = "https://www.vstu.ru/"
RASP_PREFIX = "https://www.vstu.ru/student/raspisaniya/zanyatiy/index.php?dep="
session = requests.Session()
session.headers = CaseInsensitiveDict(
{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
"Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"Referer": "http://dump.vstu.ru/",
"Upgrade-Insecure-Requests": "1",
"Priority": "u=0, i",
"Pragma": "no-cache",
"Cache-Control": "no-cach",
}
)
EXCEL_LINKS = {}
filestime = str(round(time.time()))
for facultet in FACULTETS:
url = RASP_PREFIX + facultet
r = session.get(url)
print(f"GET {url}")
soup = BeautifulSoup(r.text, 'html.parser')
excel_pattern = re.compile(r'\.xlsx?$')
# Ищем все теги <a>, у которых атрибут href соответствует нашему паттерну
excel_tags = soup.find_all('a', href=excel_pattern)
excel_links = [tag.get('href') for tag in excel_tags]
# Предположим, вы уже получили excel_links из одного из методов выше
# excel_links = ['../../../upload/raspisanie/z/ОН_ХТФ_1 курс.xlsx', ...]
absolute_links = [urljoin(BASE_URL, relative_link) for relative_link in excel_links]
if facultet not in EXCEL_LINKS.keys():
EXCEL_LINKS[facultet] = set()
for excel_url in absolute_links:
EXCEL_LINKS[facultet].add(excel_url)
print(f"+url {excel_url}")
DEBUG_ONE_FAC = None #'fevt'
result = {}
faileds = []
counter = 0
for facultet in FACULTETS:
counter += 1000
print(f"\n\n-- Факультет '{facultet}' --")
facultet_urls = EXCEL_LINKS[facultet]
for excel_url in facultet_urls:
counter += 1
print(f"\n\n-- Ссылка --")
print(f"{excel_url}")
def process_excel_file(facultet, excel_url, counter, timeid):
is_xlsx = excel_url.endswith(".xlsx")
try:
filename = "excels/" + facultet + filestime + f"[C{counter}]" + ".xls" + ("x" if is_xlsx else "")
filename = "excels/" + timeid + "_" + facultet + f"_[C{counter}]" + ".xls" + ("x" if is_xlsx else "")
aigenerated.download_file_from_url(excel_url, filename)
reader = translations.create_reader(filename)
print("Reader info")
print(reader.info())
while True:
print(f"Parsing sheet №{reader.get_sheet_index()+1}")
parser.LOGGING = False
prs = parser.Parser(reader)
prs.parse()
@@ -98,13 +51,21 @@ for facultet in FACULTETS:
gr['data_source'] = excel_url.split("/")[-1]
gr['parser_debug'] = {
"C_COUNTER": counter,
"timestime": filestime,
"timeid": timeid,
"excel_url": excel_url,
"reader_info": reader.info(),
"reader_sheet_index": reader.get_sheet_index(),
"filename": filename
}
print(f"Populates {len(prs.groups)} groups to result")
print(f"Populates {len(prs.groups)} groups to result: " + " ".join(prs.groups.keys()))
if not reader.has_next_sheet():
print("File ended")
break
else:
reader.next_sheet()
print("Next sheet!")
except Exception as e:
print(f"Error while {excel_url}")
@@ -116,9 +77,35 @@ for facultet in FACULTETS:
"url": excel_url
})
json.dump(result, open('result.json', 'w'), indent=2, ensure_ascii=False)
print("Faileds:")
print(faileds)
result = {}
faileds = []
def main():
EXCEL_LINKS = links_parser.parse_links(FACULTETS if DEBUG_ONE_FAC is None else [DEBUG_ONE_FAC])
print("Saved to result.json")
counter = 0
timeid = str(round(time.time()))
for facultet in EXCEL_LINKS.keys():
counter += 1000
print(f"\n\n-- Факультет '{facultet}' --")
facultet_urls = EXCEL_LINKS[facultet]
for excel_url in facultet_urls:
counter += 1
print(f"\n\n-- Ссылка --")
print(f"{excel_url}")
print("Start processing excel file")
process_excel_file(facultet, excel_url, counter, timeid)
print("Excel file processing done!")
print("Saving result.json")
json.dump(result, open('result.json', 'w'), indent=2, ensure_ascii=False)
print("Saved to result.json")
print("Faileds:")
print(faileds)
if __name__ == "__main__":
main()
print("Bye!")

270254
result.json
View File

File diff suppressed because it is too large Load Diff

60
translations.py
View File

@@ -15,7 +15,7 @@ class TranschendentnostCell:
self._is_empty = is_empty
def is_empty(self):
self._is_empty
return self._is_empty
class ExcelSheetReader(ABC):
"""
@@ -25,6 +25,18 @@ class ExcelSheetReader(ABC):
def __init__(self, file_path):
self.file_path = file_path
@abstractmethod
def get_sheet_index(self):
pass
@abstractmethod
def has_next_sheet(self):
pass
@abstractmethod
def next_sheet(self):
pass
@abstractmethod
def get_cell_value(self, row, col):
"""Возвращает значение ячейки по 0-индексированным координатам."""
@@ -96,8 +108,23 @@ class ExcelSheetReader(ABC):
class XlrdSheetReader(ExcelSheetReader):
def __init__(self, file_path, sheet_index=0):
super().__init__(file_path)
self.sheet_index = sheet_index
self.book = xlrd.open_workbook(file_path, formatting_info=True)
self.sheet = self.book.sheet_by_index(sheet_index)
self.init_sheet()
def get_sheet_index(self):
return self.sheet_index
def init_sheet(self):
self.sheet = self.book.sheet_by_index(self.sheet_index)
def has_next_sheet(self):
return self.sheet_index < len(self.book.sheet_names())-1
def next_sheet(self):
if self.has_next_sheet():
self.sheet_index += 1
self.init_sheet()
def get_cell_value(self, row, col):
# Проверка на выход за пределы таблицы, чтобы избежать ошибок
@@ -106,9 +133,9 @@ class XlrdSheetReader(ExcelSheetReader):
return None
def info(self):
print("The number of worksheets is {0}".format(self.book.nsheets))
print("Worksheet name(s): {0}".format(self.book.sheet_names()))
return "'{0}': size: {1}x{2} names: ".format(self.sheet.name, self.sheet.nrows, self.sheet.ncols, " ".join(self.book.sheet_names()))
return """[XLRD (.xls)] The number of worksheets is {0}
Worksheet name(s): {1}
'{2}': size: {3}x{4}""".format(self.book.nsheets, self.book.sheet_names(), self.sheet.name, self.sheet.nrows, self.sheet.ncols)
def cell(self, row, col):
"""Возвращает абстрактную клетку"""
@@ -150,8 +177,9 @@ class XlrdSheetReader(ExcelSheetReader):
class OpenpyxlSheetReader(ExcelSheetReader):
def __init__(self, file_path, sheet_name=None):
super().__init__(file_path)
self.sheet_index = 0
self.workbook = openpyxl.load_workbook(file_path, data_only=True)
self.sheet = self.workbook[sheet_name] if sheet_name else self.workbook.active
self.init_sheet()
# Словарь для трансляции стилей границ openpyxl в числовые коды xlrd
self.BORDER_STYLE_MAP = {
@@ -161,6 +189,26 @@ class OpenpyxlSheetReader(ExcelSheetReader):
'slantDashDot': 13
}
def get_sheet_index(self):
return self.sheet_index
def has_next_sheet(self):
return self.sheet_index < len(self.workbook.sheetnames)-1
def next_sheet(self):
if self.has_next_sheet():
self.sheet_index += 1
self.init_sheet()
def init_sheet(self):
self.sheet = self.workbook[self.workbook.sheetnames[self.sheet_index]]
def info(self):
return """[OpenPyXL (.xlsx)] The number of worksheets is {0}
Worksheet name(s): {1}
'{2}': size: {3}x{4}""".format(len(self.workbook.sheetnames), self.workbook.sheetnames, self.sheet, self.sheet.max_row, self.sheet.max_column)
def _get_cell(self, row, col):
"""Внутренний метод для получения ячейки с преобразованием координат."""
# openpyxl использует 1-индексированную систему