launch fixes and more small-fixes of launcher behavior
All checks were successful
Build and Run VSTU Schedule Parser / build_and_run (push) Successful in 18s
All checks were successful
Build and Run VSTU Schedule Parser / build_and_run (push) Successful in 18s
This commit is contained in:
@@ -3,4 +3,4 @@ WORKDIR /app
|
|||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r requirements.txt
|
||||||
COPY . .
|
COPY . .
|
||||||
CMD ["python", "-u", "main.py"]
|
CMD ["sh", "-c", "while true; do echo '[sh] starting script...'; python -u main.py; echo '[sh] sleeping 900s'; sleep 900; done"]
|
||||||
@@ -1,7 +1,9 @@
|
|||||||
# Copyright Stanislav Mironov
|
# Copyright Stanislav Mironov
|
||||||
|
|
||||||
|
|
||||||
|
import random
|
||||||
import re
|
import re
|
||||||
|
import time
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
import requests
|
import requests
|
||||||
from requests.structures import CaseInsensitiveDict
|
from requests.structures import CaseInsensitiveDict
|
||||||
@@ -56,6 +58,9 @@ def parse_links(facultets):
|
|||||||
}
|
}
|
||||||
print("Found in vstu.ru: ", record)
|
print("Found in vstu.ru: ", record)
|
||||||
EXCEL_LINKS.append(record)
|
EXCEL_LINKS.append(record)
|
||||||
|
st = random.randint(1, 10)/10
|
||||||
|
print(f"sleep {st}s")
|
||||||
|
time.sleep(st)
|
||||||
|
|
||||||
return sorted(EXCEL_LINKS, key=lambda x: x['url'])
|
return sorted(EXCEL_LINKS, key=lambda x: x['url'])
|
||||||
|
|
||||||
|
|||||||
35
main.py
35
main.py
@@ -10,6 +10,8 @@ import random
|
|||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
|
from pika.exceptions import ChannelWrongStateError
|
||||||
import parser
|
import parser
|
||||||
import translations
|
import translations
|
||||||
import utils
|
import utils
|
||||||
@@ -21,6 +23,8 @@ load_dotenv()
|
|||||||
|
|
||||||
RABBITMQ_URL = os.environ.get("RABBITMQ_URL")
|
RABBITMQ_URL = os.environ.get("RABBITMQ_URL")
|
||||||
EXCHANGE_NAME = os.environ.get("RABBITMQ_EXCHANGE", "vstu_schedule")
|
EXCHANGE_NAME = os.environ.get("RABBITMQ_EXCHANGE", "vstu_schedule")
|
||||||
|
INFINITY_LOOP = os.environ.get("INFINITY_LOOP", "no").lower() in ['yes', "true"]
|
||||||
|
parser.LOGGING = LOGGING = os.environ.get("PARSER_LOGGING", "no").lower() in ['yes', "true"]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
connection = pika.BlockingConnection(pika.URLParameters(RABBITMQ_URL))
|
connection = pika.BlockingConnection(pika.URLParameters(RABBITMQ_URL))
|
||||||
@@ -45,8 +49,6 @@ PARSED_DIR = "parsed"
|
|||||||
DEBUG_ONE_FAC = None #'fevt'
|
DEBUG_ONE_FAC = None #'fevt'
|
||||||
DEBUG_NO_SAVE_STATES = False
|
DEBUG_NO_SAVE_STATES = False
|
||||||
|
|
||||||
parser.LOGGING = LOGGING = True
|
|
||||||
|
|
||||||
def parse_sheets(download_place):
|
def parse_sheets(download_place):
|
||||||
to_return = {}
|
to_return = {}
|
||||||
try:
|
try:
|
||||||
@@ -206,7 +208,7 @@ def run_session():
|
|||||||
"excel_dict": excel_dict
|
"excel_dict": excel_dict
|
||||||
}, ensure_ascii=False).encode('utf-8')
|
}, ensure_ascii=False).encode('utf-8')
|
||||||
)
|
)
|
||||||
print(f"RabbitMQ published r={r}")
|
print(f"RabbitMQ published 'parser.excel_found.new'")
|
||||||
new_files += 1
|
new_files += 1
|
||||||
|
|
||||||
else:
|
else:
|
||||||
@@ -266,7 +268,7 @@ def run_session():
|
|||||||
delivery_mode=2
|
delivery_mode=2
|
||||||
),
|
),
|
||||||
body=json.dumps({
|
body=json.dumps({
|
||||||
"type": "excel_file_parsed",
|
"type": "excel_file_parsed_not_same",
|
||||||
"is_new": is_new,
|
"is_new": is_new,
|
||||||
"state": state
|
"state": state
|
||||||
}, ensure_ascii=False).encode('utf-8')
|
}, ensure_ascii=False).encode('utf-8')
|
||||||
@@ -276,6 +278,9 @@ def run_session():
|
|||||||
states.append(state)
|
states.append(state)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
if isinstance(e, ChannelWrongStateError):
|
||||||
|
raise e
|
||||||
|
|
||||||
faileds.append({
|
faileds.append({
|
||||||
"uuid": str(uuid.uuid4()),
|
"uuid": str(uuid.uuid4()),
|
||||||
"exception": str(e),
|
"exception": str(e),
|
||||||
@@ -326,8 +331,7 @@ def run_session():
|
|||||||
delivery_mode=2
|
delivery_mode=2
|
||||||
),
|
),
|
||||||
body=json.dumps({
|
body=json.dumps({
|
||||||
"type": "schedule_result_v2",
|
"type": "schedule_result_v2_changed",
|
||||||
"data": d
|
|
||||||
}, ensure_ascii=False).encode('utf-8')
|
}, ensure_ascii=False).encode('utf-8')
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -338,7 +342,7 @@ def run_session():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error deleting directory '{DIRNAME}': {e}")
|
print(f"Error deleting directory '{DIRNAME}': {e}")
|
||||||
|
|
||||||
return {"changed_files": changed_files, 'total_files': total_files, "changed": changed, "new_files": new_files}
|
return {"changed_files": changed_files, 'total_files': total_files, "changed": changed, "new_files": new_files, "faileds": faileds, "faileds_count": len(faileds)}
|
||||||
|
|
||||||
def check_dirs():
|
def check_dirs():
|
||||||
|
|
||||||
@@ -346,7 +350,11 @@ def check_dirs():
|
|||||||
os.mkdir(PARSED_DIR)
|
os.mkdir(PARSED_DIR)
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
while True:
|
flag = True
|
||||||
|
while flag:
|
||||||
|
if not INFINITY_LOOP:
|
||||||
|
flag = False
|
||||||
|
|
||||||
t = utils.StepTimeCounter()
|
t = utils.StepTimeCounter()
|
||||||
err = None
|
err = None
|
||||||
sess = None
|
sess = None
|
||||||
@@ -380,11 +388,12 @@ def main():
|
|||||||
"session": sess
|
"session": sess
|
||||||
}, ensure_ascii=False).encode('utf-8')
|
}, ensure_ascii=False).encode('utf-8')
|
||||||
)
|
)
|
||||||
|
|
||||||
sleep_time = random.randint(14*60, 21*60)
|
if flag:
|
||||||
print(f"Sleep for {round(sleep_time/6)/10} minutes")
|
sleep_time = random.randint(14*60, 21*60)
|
||||||
time.sleep(sleep_time)
|
print(f"Sleep for {round(sleep_time/6)/10} minutes")
|
||||||
print("Wake up!")
|
time.sleep(sleep_time)
|
||||||
|
print("Wake up!")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user