refactor converter

- split book class into html-preprocessor, json-converter, book(with main flow) classes.
- pick out logging, law carta setup, updating status via api into separate objects
- in html-preprocesser: add cleaning hrefs
This commit is contained in:
shirshasa
2020-08-28 11:58:04 +03:00
parent b41e86b9b2
commit f27eefb96b
5 changed files with 961 additions and 908 deletions

View File

@@ -15,7 +15,7 @@ from book import Book
def configure_file_logger(name, filename='logs/converter_log.log', filemode='w+',
logging_level=logging.INFO,
logging_format='%(asctime)s - %(message)s'):
logging_format='%(asctime)s - %(levelname)s - %(message)s'):
logger = logging.getLogger(name)
folder_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -32,11 +32,10 @@ def configure_file_logger(name, filename='logs/converter_log.log', filemode='w+'
def convert_book(book_id, access, logger, libra_locker):
logger.info(f'Start processing book-{book_id}.')
logging_format = '%(asctime)s - %(levelname)s - %(message)s'
try:
book = Book(book_id, access, main_logger=logger, libra_locker=libra_locker)
book.conversion(logging_format=logging_format)
book.conversion()
except Exception as exc:
raise exc
@@ -105,7 +104,7 @@ def local_run(books):
def server_run():
logger = configure_file_logger('consumer', logging_format='%(asctime)s - %(levelname)s - %(message)s')
logger = configure_file_logger('consumer')
folder_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
config_path = Path(os.path.join(folder_path, "config/queue_config.json"))