work

.gitignore

@@ -1,4 +1,6 @@
 *.xls
 *.xlsx
 __pycache__
-.idea
+.idea
+result.json
+groups.json

links_parser.py

@@ -0,0 +1,53 @@
+import re
+import time
+from urllib.parse import urljoin
+import requests
+from requests.structures import CaseInsensitiveDict
+from bs4 import BeautifulSoup
+
+BASE_URL = "https://www.vstu.ru/"
+RASP_PREFIX = "https://www.vstu.ru/student/raspisaniya/zanyatiy/index.php?dep="
+
+
+def parse_links(facultets):
+    session = requests.Session()
+    session.headers = CaseInsensitiveDict(
+        {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+            "Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
+            "Accept-Encoding": "gzip, deflate",
+            "Connection": "keep-alive",
+            "Referer": "http://dump.vstu.ru/",
+            "Upgrade-Insecure-Requests": "1",
+            "Priority": "u=0, i",
+            "Pragma": "no-cache",
+            "Cache-Control": "no-cach",
+        }
+    )
+
+    EXCEL_LINKS = {}
+    for facultet in facultets:
+        url = RASP_PREFIX + facultet
+        r = session.get(url)
+        print(f"GET {url}")
+        soup = BeautifulSoup(r.text, 'html.parser')
+        excel_pattern = re.compile(r'\.xlsx?$')
+
+        # Ищем все теги <a>, у которых атрибут href соответствует нашему паттерну
+        excel_tags = soup.find_all('a', href=excel_pattern)
+        excel_links = [tag.get('href') for tag in excel_tags]
+
+        # Предположим, вы уже получили excel_links из одного из методов выше
+        # excel_links = ['../../../upload/raspisanie/z/ОН_ХТФ_1 курс.xlsx', ...]
+
+        absolute_links = [urljoin(BASE_URL, relative_link) for relative_link in excel_links]
+
+        if facultet not in EXCEL_LINKS.keys():
+            EXCEL_LINKS[facultet] = set()
+
+        for excel_url in absolute_links:
+            EXCEL_LINKS[facultet].add(excel_url)
+            print(f"+url {excel_url}")
+    
+    return EXCEL_LINKS

main.py

@@ -8,7 +8,7 @@ import xlwt
 
 import xlrd
 import requests
-from requests.structures import CaseInsensitiveDict
+
 
 from bs4 import BeautifulSoup
 import aigenerated
@@ -16,75 +16,28 @@ import parser
 import translations
 import utils
 import json
+import links_parser
 # Общее правило проекта, сначала в координатах идёт ROW а потом COL, нумерация с нуля
 
 FACULTETS = [
     "asp", "mag", "fastiv", "fat", "ftkm", "ftpp", "feu", "fevt", "htf", "vkf", "mmf", "fpik"
 ]
-BASE_URL = "https://www.vstu.ru/"
-RASP_PREFIX = "https://www.vstu.ru/student/raspisaniya/zanyatiy/index.php?dep="
 
-session = requests.Session()
-session.headers = CaseInsensitiveDict(
-    {
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
-        "Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
-        "Accept-Encoding": "gzip, deflate",
-        "Connection": "keep-alive",
-        "Referer": "http://dump.vstu.ru/",
-        "Upgrade-Insecure-Requests": "1",
-        "Priority": "u=0, i",
-        "Pragma": "no-cache",
-        "Cache-Control": "no-cach",
-    }
-)
-
-EXCEL_LINKS = {}
-filestime = str(round(time.time()))
-for facultet in FACULTETS:
-    url = RASP_PREFIX + facultet
-    r = session.get(url)
-    print(f"GET {url}")
-    soup = BeautifulSoup(r.text, 'html.parser')
-    excel_pattern = re.compile(r'\.xlsx?$')
-
-    # Ищем все теги <a>, у которых атрибут href соответствует нашему паттерну
-    excel_tags = soup.find_all('a', href=excel_pattern)
-    excel_links = [tag.get('href') for tag in excel_tags]
-
-    # Предположим, вы уже получили excel_links из одного из методов выше
-    # excel_links = ['../../../upload/raspisanie/z/ОН_ХТФ_1 курс.xlsx', ...]
-
-    absolute_links = [urljoin(BASE_URL, relative_link) for relative_link in excel_links]
-
-    if facultet not in EXCEL_LINKS.keys():
-        EXCEL_LINKS[facultet] = set()
-
-    for excel_url in absolute_links:
-        EXCEL_LINKS[facultet].add(excel_url)
-        print(f"+url {excel_url}")
+DEBUG_ONE_FAC = None #'fevt'
 
 
-result = {}
-faileds = []
-counter = 0
-for facultet in FACULTETS:
-    counter += 1000
-    print(f"\n\n-- Факультет '{facultet}' --")
-    facultet_urls = EXCEL_LINKS[facultet]
-    for excel_url in facultet_urls:
-        counter += 1
-        print(f"\n\n-- Ссылка --")
-        print(f"{excel_url}")
-        is_xlsx = excel_url.endswith(".xlsx")
+def process_excel_file(facultet, excel_url, counter, timeid):
+    is_xlsx = excel_url.endswith(".xlsx")
+    try:
+        filename = "excels/" + timeid + "_" + facultet + f"_[C{counter}]" + ".xls" + ("x" if is_xlsx else "")
+        aigenerated.download_file_from_url(excel_url, filename)
 
-        try:
-            filename = "excels/" + facultet + filestime + f"[C{counter}]" + ".xls" + ("x" if is_xlsx else "")
-            aigenerated.download_file_from_url(excel_url, filename)
-
-            reader = translations.create_reader(filename)
+        reader = translations.create_reader(filename)
+        print("Reader info")
+        print(reader.info())
 
+        while True:
+            print(f"Parsing sheet №{reader.get_sheet_index()+1}")
             parser.LOGGING = False
             prs = parser.Parser(reader)
             prs.parse()
@@ -92,33 +45,67 @@ for facultet in FACULTETS:
                 if group_name in result.keys():
                     print(f" -- WTF -- Doubled groups -- name: {group_name}")
                     continue
-                
+                        
                 gr = result[group_name] = prs.groups[group_name]
                 gr['facultet'] = facultet
                 gr['data_source'] = excel_url.split("/")[-1]
                 gr['parser_debug'] = {
                     "C_COUNTER": counter,
-                    "timestime": filestime,
+                    "timeid": timeid,
                     "excel_url": excel_url,
                     "reader_info": reader.info(),
+                    "reader_sheet_index": reader.get_sheet_index(),
                     "filename": filename
                 }
 
-            print(f"Populates {len(prs.groups)} groups to result")
+            print(f"Populates {len(prs.groups)} groups to result: " + " ".join(prs.groups.keys()))
 
-        except Exception as e:
-            print(f"Error while {excel_url}")
-            print(e)
-            traceback.print_exc()
-            faileds.append({
-                "ex": e,
-                "fac": facultet,
-                "url": excel_url
-            })
+            if not reader.has_next_sheet():
+                print("File ended")
+                break
+            else:
+                reader.next_sheet()
+                print("Next sheet!")
 
-json.dump(result, open('result.json', 'w'), indent=2, ensure_ascii=False)
+    except Exception as e:
+        print(f"Error while {excel_url}")
+        print(e)
+        traceback.print_exc()
+        faileds.append({
+            "ex": e,
+            "fac": facultet,
+            "url": excel_url
+        })
 
-print("Faileds:")
-print(faileds)
 
-print("Saved to result.json")
+result = {}
+faileds = []
+def main():
+    EXCEL_LINKS = links_parser.parse_links(FACULTETS if DEBUG_ONE_FAC is None else [DEBUG_ONE_FAC])
+
+    counter = 0
+    timeid = str(round(time.time()))
+    for facultet in EXCEL_LINKS.keys():
+        counter += 1000
+        print(f"\n\n-- Факультет '{facultet}' --")
+        facultet_urls = EXCEL_LINKS[facultet]
+        for excel_url in facultet_urls:
+            counter += 1
+            print(f"\n\n-- Ссылка --")
+            print(f"{excel_url}")
+            
+            print("Start processing excel file")
+            process_excel_file(facultet, excel_url, counter, timeid)
+            print("Excel file processing done!")
+
+    print("Saving result.json")
+    json.dump(result, open('result.json', 'w'), indent=2, ensure_ascii=False)
+    print("Saved to result.json")
+
+    print("Faileds:")
+    print(faileds)
+
+
+if __name__ == "__main__":
+    main()
+    print("Bye!")

translations.py

@@ -15,7 +15,7 @@ class TranschendentnostCell:
         self._is_empty = is_empty
 
     def is_empty(self):
-        self._is_empty
+        return self._is_empty
 
 class ExcelSheetReader(ABC):
     """
@@ -25,6 +25,18 @@ class ExcelSheetReader(ABC):
     def __init__(self, file_path):
         self.file_path = file_path
 
+    @abstractmethod
+    def get_sheet_index(self):
+        pass
+
+    @abstractmethod
+    def has_next_sheet(self):
+        pass
+
+    @abstractmethod
+    def next_sheet(self):
+        pass
+
     @abstractmethod
     def get_cell_value(self, row, col):
         """Возвращает значение ячейки по 0-индексированным координатам."""
@@ -96,8 +108,23 @@ class ExcelSheetReader(ABC):
 class XlrdSheetReader(ExcelSheetReader):
     def __init__(self, file_path, sheet_index=0):
         super().__init__(file_path)
+        self.sheet_index = sheet_index
         self.book = xlrd.open_workbook(file_path, formatting_info=True)
-        self.sheet = self.book.sheet_by_index(sheet_index)
+        self.init_sheet()
+        
+    def get_sheet_index(self):
+        return self.sheet_index
+
+    def init_sheet(self):
+        self.sheet = self.book.sheet_by_index(self.sheet_index)
+
+    def has_next_sheet(self):
+        return self.sheet_index < len(self.book.sheet_names())-1
+
+    def next_sheet(self):
+        if self.has_next_sheet():
+            self.sheet_index += 1
+            self.init_sheet()
 
     def get_cell_value(self, row, col):
         # Проверка на выход за пределы таблицы, чтобы избежать ошибок
@@ -106,9 +133,9 @@ class XlrdSheetReader(ExcelSheetReader):
         return None
     
     def info(self):
-        print("The number of worksheets is {0}".format(self.book.nsheets))
-        print("Worksheet name(s): {0}".format(self.book.sheet_names()))
-        return "'{0}': size: {1}x{2} names: ".format(self.sheet.name, self.sheet.nrows, self.sheet.ncols, " ".join(self.book.sheet_names()))
+        return """[XLRD (.xls)] The number of worksheets is {0}
+        Worksheet name(s): {1}
+        '{2}': size: {3}x{4}""".format(self.book.nsheets, self.book.sheet_names(), self.sheet.name, self.sheet.nrows, self.sheet.ncols)
     
     def cell(self, row, col):
         """Возвращает абстрактную клетку"""
@@ -150,8 +177,9 @@ class XlrdSheetReader(ExcelSheetReader):
 class OpenpyxlSheetReader(ExcelSheetReader):
     def __init__(self, file_path, sheet_name=None):
         super().__init__(file_path)
+        self.sheet_index = 0
         self.workbook = openpyxl.load_workbook(file_path, data_only=True)
-        self.sheet = self.workbook[sheet_name] if sheet_name else self.workbook.active
+        self.init_sheet()
 
         # Словарь для трансляции стилей границ openpyxl в числовые коды xlrd
         self.BORDER_STYLE_MAP = {
@@ -161,6 +189,26 @@ class OpenpyxlSheetReader(ExcelSheetReader):
             'slantDashDot': 13
         }
 
+    def get_sheet_index(self):
+        return self.sheet_index
+    
+    def has_next_sheet(self):
+        return self.sheet_index < len(self.workbook.sheetnames)-1
+
+    def next_sheet(self):
+        if self.has_next_sheet():
+            self.sheet_index += 1
+            self.init_sheet()
+
+    def init_sheet(self):
+        self.sheet = self.workbook[self.workbook.sheetnames[self.sheet_index]]
+
+    def info(self):
+        return """[OpenPyXL (.xlsx)] The number of worksheets is {0}
+        Worksheet name(s): {1}
+        '{2}': size: {3}x{4}""".format(len(self.workbook.sheetnames), self.workbook.sheetnames, self.sheet, self.sheet.max_row, self.sheet.max_column)
+    
+    
     def _get_cell(self, row, col):
         """Внутренний метод для получения ячейки с преобразованием координат."""
         # openpyxl использует 1-индексированную систему