lxbfYeaa/BookConverter
Archived
Template
1
0
Fork 0
forked from LiveCarta/BookConverter
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change structure of docx converter

Browse Source
This commit is contained in:
Kiryl
2021-09-30 13:33:08 +03:00
parent af1b6138a9
commit cc977a7167
6 changed files with 10 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
src/docx_converter/docx_solver.py Normal file
View File

@@ -0,0 +1,158 @@
import logging
import os
import pathlib
import subprocess
from subprocess import PIPE
from threading import Event
from bs4 import BeautifulSoup
from html_docx_preprocessor import HTMLDocxPreprocessor
from libra_html2json_converter import LibraHTML2JSONConverter
from src.book_solver import BookSolver
class DocxBook(BookSolver):
def __init__(self, book_id=0, access=None, html_path=None,
main_logger=None, libra_locker=None, logging_format='%(asctime)s - %(levelname)s - %(message)s'):
super().__init__(book_id, access, main_logger, logging_format)
self.book_type = 'docx'
self.html_path = html_path # path to html file, file appears after libre-conversion
self.libra_locker: Event() = libra_locker # critical section for occupying libreoffice by one thread
def _libra_run(self, out_dir_path):
command = ['libreoffice', '--headless',
'--convert-to', 'html', f'{str(self.file_path)}',
'--outdir', f'{out_dir_path}']
result = subprocess.run(command, stdout=PIPE, stderr=PIPE)
self.logger_object.log(f'Result of libra conversion for book_{self.book_id}: {result.returncode}, {result.stdout}', logging.DEBUG)
self.logger_object.log(f'Any error while libra conversion for book_{self.book_id}: {result.stderr}', logging.DEBUG)
def convert_doc_to_html(self):
"""
Method for convert .docx document to .html file.
"""
self.logger_object.log(f'File - {self.file_path}.')
print(f'{self.file_path}')
self.logger_object.log('Beginning of conversion from .docx to .html.')
try:
f = open(self.file_path)
f.close()
except FileNotFoundError as error:
self.logger_object.log('Invalid path to input data.', logging.ERROR)
self.status_wrapper.set_error()
raise error
folder_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
out_dir_path = os.path.join(folder_path, f'html/{self.book_id}')
pathlib.Path(out_dir_path).mkdir(parents=True, exist_ok=True)
is_book_converted = False
try:
if self.libra_locker.isSet():
self.libra_locker.clear()
self.logger_object.log('Got flag...', logging.DEBUG)
self._libra_run(out_dir_path)
self.libra_locker.set()
self.logger_object.log('Cleared flag...', logging.DEBUG)
else:
while not self.libra_locker.isSet() and not is_book_converted:
self.logger_object.log('Waiting for libra...', logging.DEBUG)
flag = self.libra_locker.wait(50)
if flag:
if self.libra_locker.isSet():
self.libra_locker.clear()
self.logger_object.log(f'Got flag!', logging.DEBUG)
self._libra_run(out_dir_path)
self.libra_locker.set()
break
except Exception as exc:
self.logger_object.log("Conversion has gone wrong. Libreoffice is not installed.", logging.ERROR)
self.logger_object.log_error_to_main_log()
self.status_wrapper.set_error()
raise exc
out_dir_path = os.path.join(out_dir_path, f'{self.book_id}.html')
self.html_path = pathlib.Path(out_dir_path)
try:
f = open(self.html_path)
f.close()
except FileNotFoundError as exc:
self.logger_object.log("Conversion has gone wrong. HTML file doesn't exist.", logging.ERROR)
self.logger_object.log_error_to_main_log()
self.status_wrapper.set_error()
raise exc
self.logger_object.log('End of conversion from .docx to .html.')
self.logger_object.log(f'Input file path after conversion: {self.html_path}.')
def read_html(self):
"""
Method for reading .html file into beautiful soup tag.
"""
try:
html_text = open(self.html_path, 'r', encoding='utf8').read()
self.logger_object.log('HTML for book has been loaded.')
except FileNotFoundError as exc:
self.logger_object.log('There is no html to process.'
'Conversion went wrong or you specified wrong paths.', logging.ERROR)
self.logger_object.log_error_to_main_log()
self.status_wrapper.set_error()
raise exc
html_soup = BeautifulSoup(html_text, features='lxml')
return html_soup
def write_html_from_list(self, body_tag, file_name='json/html_test.html'):
folder_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
file_path = pathlib.Path(os.path.join(folder_path, file_name))
with open(file_path, 'w', encoding='utf-8') as f_out:
f_out.write(body_tag.prettify())
self.logger_object.log(f'Check final prettified html: {file_name}.')
def convert_from_html(self):
html_soup = self.read_html()
parser = HTMLDocxPreprocessor(html_soup, self.logger_object)
content, footnotes, top_level_headers = parser.process_html(self.access, self.html_path, self.book_id)
json_converter = LibraHTML2JSONConverter(content, footnotes, top_level_headers, self.logger_object, self.status_wrapper)
content_dict = json_converter.convert_to_dict()
self.write_to_json(content_dict)
self.write_html_from_list(parser.body_tag)
def get_converted_book(self):
"""
1. Convert docx to html with libra office
2. Parse and clean html, get list of tags, get footnotes
3. Parse from line structure to nested structure with JSONConverter
"""
self.convert_doc_to_html()
self.check_output_directory()
html_soup = self.read_html()
self.logger_object.log('Beginning of processing .html file.')
parser = HTMLDocxPreprocessor(html_soup, self.logger_object)
bs_tags, footnotes, top_level_headers = parser.process_html(self.access, self.html_path, self.book_id)
self.logger_object.log('Beginning of processing json output.')
self.status_wrapper.set_generating()
json_converter = LibraHTML2JSONConverter(bs_tags, footnotes, top_level_headers, self.logger_object, self.status_wrapper)
content_dict = json_converter.convert_to_dict()
return content_dict
if __name__ == "__main__":
folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
file = pathlib.Path(os.path.join(folder, 'html/ch13/Ch_13_edit.html'))
out_path = pathlib.Path(os.path.join(folder, 'json/ch13.json'))
book = DocxBook(html_path=file)
book.convert_from_html()

695
src/docx_converter/html_docx_preprocessor.py Normal file
View File

@@ -0,0 +1,695 @@
import logging
import os
import pathlib
import re
from shutil import copyfile
from typing import List
from bs4 import BeautifulSoup, NavigableString, Tag
from src.livecarta_config import LiveCartaConfig
from src.util.helpers import BookLogger, BookStatusWrapper
class HTMLDocxPreprocessor:
def __init__(self, html_soup, logger_object, status_wrapper=None):
self.body_tag = html_soup.body
self.html_soup = html_soup
self.logger_object: BookLogger = logger_object
self.status_wrapper: BookStatusWrapper = status_wrapper
self.top_level_headers = None
self.content = list()
def _clean_tag(self, tag, attr_name, attr_value):
"""
Function to clean tags by its name and attribute value.
:param tag: Tag name to clean.
:param attr_name: Attribute name.
:param attr_value: Attribute value.
"""
tags = self.body_tag.find_all(tag, {attr_name: attr_value})
for tag in tags:
if len(tag.attrs) == 1:
tag.unwrap()
def _clean_underline_links(self):
"""
Function cleans meaningless <u> tags before links.
"""
underlines = self.body_tag.find_all("u")
for u in underlines:
if u.find_all('a'):
u.unwrap()
links = self.body_tag.find_all('a')
for link in links:
u = link.find_all('u')
if u and len(u) == 1:
u[0].unwrap()
@classmethod
def convert_pt_to_px(cls, value):
value = float(value)
if value == LiveCartaConfig.WORD_DEFAULT_FONT_SIZE:
return LiveCartaConfig.LIVECARTA_DEFAULT_FONT_SIZE
else:
return value
@classmethod
def convert_font_pt_to_px(cls, style):
"""
Method converts point in the font-size to pixels.
:param style: Str with style to process.
:return: Str with converted style.
"""
size = re.search(r"font-size: (\d{1,3})pt", style)
if size is None:
return style
size = size.group(1)
new_size = cls.convert_pt_to_px(size)
if new_size == LiveCartaConfig.LIVECARTA_DEFAULT_FONT_SIZE:
return ""
return re.sub(size + "pt", str(new_size) + "px", style)
def _font_to_span(self):
"""
Function to convert <font> tag to <span>. If font style is default, then remove this tag.
"""
fonts = self.body_tag.find_all("font")
for font in fonts:
face = font.get("face")
style = font.get("style")
color = font.get("color")
font.attrs = {}
font.name = "span"
if style:
style = self.convert_font_pt_to_px(style)
if style != "":
if color and color in LiveCartaConfig.COLORS_MAP:
style += f'; color: {color};'
font.attrs["style"] = style
elif color and color in LiveCartaConfig.COLORS_MAP:
font.attrs["style"] = f'color: {color};'
if face is not None:
face = re.sub(r",[\w,\- ]*$", "", face)
if face != LiveCartaConfig.DEFAULT_FONT_NAME and LiveCartaConfig.font_correspondence_table.get(face):
font.attrs["face"] = LiveCartaConfig.font_correspondence_table[face]
else:
font.attrs["face"] = LiveCartaConfig.DEFAULT_FONT_NAME
if len(font.attrs) == 0:
font.unwrap()
assert len(self.body_tag.find_all("font")) == 0 # on this step there should be no more <font> tags
def delete_content_before_toc(self):
# remove all tag upper the <TOC> only in content !!! body tag is not updated
toc_tag = self.html_soup.new_tag('TOC')
if toc_tag in self.content:
ind = self.content.index(toc_tag) + 1
self.content = self.content[ind:]
def clean_trash(self):
"""
Function to remove all styles and tags we don't need.
"""
self._clean_tag('span', 'style', re.compile(r'^background: #[0-9a-fA-F]{6}$'))
self._clean_tag('span', 'lang', re.compile(r'^ru-RU$')) # todo: check for another languages
self._clean_tag('span', 'style', re.compile('^letter-spacing: -?[\d\.]+pt$'))
self._clean_tag('font', 'face', re.compile(r'^Times New Roman[\w, ]+$'))
self._clean_tag("a", "name", "_GoBack")
self._clean_underline_links()
self._font_to_span()
# replace toc with empty <TOC> tag
tables = self.body_tag.find_all("div", id=re.compile(r'^Table of Contents\d+'))
for table in tables:
table.wrap(self.html_soup.new_tag("TOC"))
table.decompose()
def _process_paragraph(self):
"""
Function to process <p> tags (text-align and text-indent value).
"""
paragraphs = self.body_tag.find_all('p')
for p in paragraphs:
# libra converts some \n into <p> with 2 </br>
# there we remove 1 unnecessary <br>
brs = p.find_all('br')
text = p.text
if brs and text == '\n\n' and len(brs) == 2:
brs[0].decompose()
indent_should_be_added = False
if text and ((text[0:1] == '\t') or (text[:2] == '\n\t')):
indent_should_be_added = True
align = p.get('align')
style = p.get('style')
if style:
indent = re.search(r'text-indent: ([\d\.]{1,4})in', style)
margin_left = re.search(r'margin-left: ([\d\.]{1,4})in', style)
margin_right = re.search(r'margin-right: ([\d\.]{1,4})in', style)
margin_top = re.search(r'margin-top: ([\d\.]{1,4})in', style)
margin_bottom = re.search(r'margin-bottom: ([\d\.]{1,4})in', style)
else:
indent = None
margin_left = None
margin_right = None
margin_top = None
margin_bottom = None
if margin_left and margin_right and margin_top and margin_bottom and \
margin_left.group(1) == '0.6' and margin_right.group(1) == '0.6' and \
margin_top.group(1) == '0.14' and margin_bottom.group(1) == '0.11':
p.wrap(BeautifulSoup(features='lxml').new_tag('blockquote'))
p.attrs = {}
style = ''
if align is not None and align != LiveCartaConfig.DEFAULT_ALIGN_STYLE:
style += f'text-align: {align};'
if indent is not None or indent_should_be_added:
# indent = indent.group(1)
style += f'text-indent: {LiveCartaConfig.INDENT};'
if style:
p.attrs['style'] = style
def _process_two_columns(self):
"""
Function to process paragraphs which has two columns layout.
"""
two_columns = self.body_tag.find_all("div", style="column-count: 2")
for div in two_columns:
for child in div.children:
if child.name == "p":
child["class"] = "columns2"
div.unwrap()
def _process_tables(self):
"""
Function to process tables. Set "border" attribute.
"""
tables = self.body_tag.find_all("table")
for table in tables:
tds = table.find_all("td")
sizes = []
for td in tds:
style = td.get('style')
if style:
match = re.search(r"border: ?(\d+\.?\d*)(p[tx])", style)
if match:
size = match.group(1)
units = match.group(2)
if units == "pt":
size = self.convert_pt_to_px(size)
sizes.append(float(size))
width = td.get('width')
td.attrs = {}
if width:
td.attrs['width'] = width
if sizes:
border_size = sum(sizes) / len(sizes)
table.attrs['border'] = f'{border_size:.2}'
self.tables_amount = len(tables)
def _process_quotes(self):
"""
Function to process block quotes.
After docx to html conversion block quotes are stored inside table with 1 cell.
All text is wrapped in a <i> tag.
Such tables will be replaced with <blockquote> tags.
<table cellpadding=\"7\" cellspacing=\"0\" width=\"614\">
<col width=\"600\"/>
<tr>
<td width=\"600\">
<p style=\"text-align: justify;\"><i>aaaaa</i></p>
<p style=\"text-align: justify;\"><br/></p>
</td>
</tr>
</table>
"""
tables = self.body_tag.find_all("table")
for table in tables:
trs = table.find_all("tr")
tds = table.find_all("td")
if len(trs) == 1 and len(tds) == 1 and tds[0].get('width') == '600':
td = tds[0]
is_zero_border = 'border: none;' in td.get('style')
paragraphs = td.find_all("p")
has_i_tag_or_br = [(p.i, p.br) for p in paragraphs]
has_i_tag_or_br = [x[0] is not None or x[1] is not None
for x in has_i_tag_or_br]
if all(has_i_tag_or_br) and is_zero_border:
new_div = BeautifulSoup(features='lxml').new_tag('blockquote')
for p in paragraphs:
new_div.append(p)
table.replaceWith(new_div)
def _process_hrefs(self):
a_tags_with_href = self.body_tag.find_all('a', {'href': re.compile('^.*http.+')})
# remove char=end of file for some editors
for tag in a_tags_with_href:
tag.string = tag.text.replace('\u200c', '')
tag['href'] = tag.attrs.get('href').replace('%E2%80%8C', '')
a_tags_with_href = self.body_tag.find_all('a', {'href': re.compile('^(?!#sdfootnote)')})
for tag in a_tags_with_href:
tag.string = tag.text.replace('\u200c', '')
tag.string = tag.text.replace('\u200b', '') # zero-width-space
tag['href'] = tag.attrs.get('href').replace('%E2%80%8C', '')
@staticmethod
def _clean_footnote_content(content):
content = content.strip()
return content.strip()
def _process_footnotes(self):
"""
Function returns list of footnotes and delete them from html_soup.
"""
footnote_anchors = self.body_tag.find_all('a', class_='sdfootnoteanc')
footnote_content = self.body_tag.find_all('div', id=re.compile(r'^sdfootnote\d+$'))
footnote_amt = len(footnote_anchors)
assert footnote_amt == len(footnote_content), \
'Some ting went wrong with footnotes after libra conversion'
footnotes = []
for i, (anc_tag, cont_tag) in enumerate(zip(footnote_anchors, footnote_content)):
true_a_tag = cont_tag.find_all('a', class_=re.compile(r'^sdfootnote.+$'))[0]
if true_a_tag.attrs.get('href') is None:
cont_tag.a.decompose()
continue
assert anc_tag['name'] == true_a_tag['href'][1:], \
'Something went wrong with footnotes after libra conversion'
new_tag = BeautifulSoup(features='lxml').new_tag('sup')
new_tag['class'] = 'footnote-element'
new_tag['data-id'] = i + 1
new_tag['id'] = f'footnote-{i + 1}'
new_tag.string = '*'
anc_tag.replace_with(new_tag)
# extra digits in footnotes from documents downloaded from livecarta
a_text = true_a_tag.text
if len(cont_tag.find_all('p')):
sup = cont_tag.find_all('p')[0].find('sup')
if sup and sup.text == a_text:
sup.decompose()
for tag_a in cont_tag.find_all('a', {'class': 'sdfootnotesym'}):
tag_a.decompose()
# remove font-size
for span in cont_tag.find_all('span', {'style': re.compile('font-size')}):
style = span.get('style')
style = re.sub(r"font-size: \d+px", "", style)
if style == '':
del span.attrs['style']
else:
span.attrs['style'] = style
unicode_string = ''
for child in cont_tag.children:
if type(child) is NavigableString:
continue
if child.name == 'blockquote':
unicode_string += str(child)
else:
unicode_string += child.decode_contents()
content = self._clean_footnote_content(unicode_string)
cont_tag.decompose()
footnotes.append(content)
self.footnotes = footnotes
def _process_images(self, access, html_path, book_id):
"""
Function to process <img> tag. Img should be sent Amazon S3 and then return new tag with valid link.
For now images are moved to one folder.
"""
img_tags = self.body_tag.find_all('img')
if len(img_tags):
if access is None:
folder_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
new_path = pathlib.Path(os.path.join(folder_path, f'json/img_{book_id}/'))
new_path.mkdir(exist_ok=True)
for img in img_tags:
img_name = img.attrs.get('src')
# quick fix for bad links
if (len(img_name) >= 3) and img_name[:3] == '../':
img_name = img_name[3:]
img_path = pathlib.Path(f'{html_path.parent}', f'{img_name}')
if access is not None:
link = access.send_image(img_path, doc_id=book_id)
img.attrs['src'] = link
self.logger_object.log(f'{img_name} successfully uploaded.')
else:
img_size = os.path.getsize(img_path)
self.logger_object.log(f'{img_name} successfully loaded. Image size: {img_size}.', logging.DEBUG)
new_img_path = new_path / img_name
copyfile(img_path, new_img_path)
img.attrs["src"] = str(new_img_path)
self.images = img_tags
def _process_footer(self):
"""
Function to process <div title="footer"> tags.
All the tags will be deleted from file.
"""
divs = self.body_tag.find_all('div', {'title': 'footer'})
for div in divs:
div.decompose()
def _process_div(self):
"""
Function to process <div> tags. All the tags will be deleted from file, all content of the tags will stay.
"""
divs = self.body_tag.find_all("div")
for div in divs:
div.unwrap()
def _check_parent_link_exist_in_toc(self, tag_with_link):
toc_links = []
for a_tag in tag_with_link.find_all("a", {'name': re.compile(r'^_Toc\d+')}):
link_name = a_tag.attrs['name']
toc_item = self.body_tag.find("a", {'href': '#' + link_name})
if toc_item:
toc_links.append(toc_item)
return len(toc_links) > 0
def _process_toc_links(self):
"""
Function to extract nodes which contains TOC links, remove links from file and detect headers.
"""
toc_links = self.body_tag.find_all("a", {'name': re.compile(r'^_Toc\d+')})
headers = [link.parent for link in toc_links]
outline_level = "1" # All the unknown outlines will be predicted as <h1>
for tag in headers:
if re.search(r"^h\d$", tag.name):
tag.a.unwrap()
# outline_level = tag.name[-1] # TODO: add prediction of the outline level
elif tag.name == "p":
exist_in_toc = self._check_parent_link_exist_in_toc(tag)
if tag in self.body_tag.find_all("p") and exist_in_toc:
new_tag = BeautifulSoup(features="lxml").new_tag("h" + outline_level)
text = tag.text
tag.replaceWith(new_tag)
new_tag.string = text
else:
# rethink document structure when you have toc_links, other cases?
self.logger_object.log(f'Something went wrong in processing toc_links.'
f' Check the structure of the file. '
f'Tag name: {tag.name}')
@staticmethod
def clean_title_from_numbering(title: str):
"""
Function to remove digits from headers.
"""
title = re.sub(r'^(\s+)+', '', title)
title = re.sub(r'^(?:\.?\d+\.? ?)+', '', title)
# title = re.sub(r'^(?:\.?[MDCLXVIclxvi]+\.? ?)+ ', '', title) # delete chapter numbering from the title
title = re.sub(r'^(?:[A-Za-z]\. ?)+', '', title)
return title
@staticmethod
def clean_tag_from_tabs(tag: NavigableString):
cleaned = re.sub(r'(\s+)+', ' ', tag)
this = BeautifulSoup.new_string(BeautifulSoup(features="lxml"), cleaned, NavigableString)
tag.replace_with(this)
# print('input: ', repr(tag))
# print('test: ', repr(cleaned))
def clean_tag_from_numbering(self, tag):
cleaned = self.clean_title_from_numbering(tag)
this = BeautifulSoup.new_string(BeautifulSoup(features="lxml"), cleaned, NavigableString)
tag.replace_with(this)
# print('input: ', repr(tag))
# print('test: ', repr(cleaned))
def apply_func_to_last_child(self, tag, func=None):
"""
works only with constructions like (((child to work with)))
where child is object of NavigableString
"""
if type(tag) is NavigableString:
func(tag)
else:
children = list(tag.children)
if children:
self.apply_func_to_last_child(children[0], func)
def _preprocessing_headings(self):
"""
Function to convert all lower level headings to p tags
"""
pattern = f'^h[{LiveCartaConfig.SUPPORTED_LEVELS + 1}-9]$'
header_tags = self.body_tag.find_all(re.compile(pattern))
for tag in header_tags:
tag.name = 'p'
def _get_top_level_headers(self):
"""
Function for gathering info about top-level chapters.
Assume:
- Headers with smallest outline(or digit in <h>) are top level chapters.
[ It is consistent with a recursive algorithm
for saving content to a resulted json structure,
which happens in header_to_json()]
"""
headers_info = []
header_tags = self.body_tag.find_all(re.compile("^h[1-9]$"))
headers_outline = [int(re.sub(r"^h", "", tag.name)) for tag in header_tags]
if headers_outline:
top_level_outline = min(headers_outline)
top_level_headers = [tag for tag in header_tags
if int(re.sub(r"^h", "", tag.name)) == top_level_outline]
for tag in top_level_headers:
if tag.parent.name == "li":
tag.parent.unwrap()
while tag.parent.name == "ol":
tag.parent.unwrap()
title = tag.text
title = re.sub(r'\s+', ' ', title).strip()
number = re.match(r'^(?:\.?\d+\.? ?)+', title)
is_numbered = number is not None
cleaned_title = self.clean_title_from_numbering(tag.text)
is_introduction = cleaned_title.lower() == 'introduction'
headers_info.append({
'title': cleaned_title,
'is_numbered': is_numbered,
'is_introduction': is_introduction})
return headers_info
def _mark_introduction_headers(self):
"""
Function to find out:
what header shouldn't be numbered and can be treated as introduction chapter
Assume header(s) to be introduction if:
1. one header not numbered, before 1 numbered header
2. it is first header from the top level list and it equals to 'introduction'
Result :
Mark each top-level header with flag should_be_numbered = true/false
"""
is_numbered_header = [header['is_numbered'] for header in self.top_level_headers]
is_title = [header['is_introduction'] for header in self.top_level_headers]
first_not_numbered = is_numbered_header and is_numbered_header[0] == 0
second_is_numbered_or_not_exist = all(is_numbered_header[1:2])
first_header_is_introduction = is_title and is_title[0]
if (first_not_numbered and second_is_numbered_or_not_exist) or first_header_is_introduction:
self.top_level_headers[0]['should_be_numbered'] = False
for i in range(1, len(self.top_level_headers)):
self.top_level_headers[i]['should_be_numbered'] = True
else:
for i in range(0, len(self.top_level_headers)):
self.top_level_headers[i]['should_be_numbered'] = True
def _process_headings(self):
"""
Function to process tags <h>.
"""
header_tags = self.body_tag.find_all(re.compile("^h[1-9]$"))
# 1. remove <b>, <span>
for tag in header_tags:
b_tags = tag.find_all("b")
[tag.unwrap() for tag in b_tags]
spans = tag.find_all("span")
if spans:
for span in spans:
style = span.attrs.get("style")
span.unwrap()
tag.attrs = {}
header_tags = self.body_tag.find_all(re.compile("^h[1-9]$"))
# 2. clean text in header from numbering and \n
for tag in header_tags:
if tag.parent.name == "li":
tag.parent.unwrap()
while tag.parent.name == "ol":
tag.parent.unwrap()
title = tag.text
title = self.clean_title_from_numbering(title)
if title == "":
tag.unwrap()
else:
assert tag.name in LiveCartaConfig.SUPPORTED_HEADERS, \
f'Preprocessing went wrong, there is still h{LiveCartaConfig.SUPPORTED_LEVELS + 1}-h9 headings.'
content = list(tag.children)
# do not take into account rubbish empty tags like <a>, but don't remove them
content = [item for item in content if
(type(item) is not NavigableString and item.text != '')
or (type(item) is NavigableString)]
for i, item in enumerate(content):
if type(content[i]) is NavigableString:
cleaned = re.sub(r'(\s+)+', ' ', content[i])
this = BeautifulSoup.new_string(BeautifulSoup(features="lxml"), cleaned, NavigableString)
content[i].replace_with(this)
content[i] = this
else:
self.apply_func_to_last_child(content[i], self.clean_tag_from_tabs)
content[0] = '' if content[0] == ' ' else content[0]
content = [item for item in content if item != '']
if type(content[0]) is NavigableString:
cleaned = self.clean_title_from_numbering(content[0])
this = BeautifulSoup.new_string(BeautifulSoup(features="lxml"), cleaned, NavigableString)
content[0].replace_with(this)
content[0] = this
else:
self.apply_func_to_last_child(content[0], self.clean_tag_from_numbering)
def _process_lists(self):
"""
Function to process tags <li>.
Unwrap <p> tags.
"""
li_tags = self.body_tag.find_all("li")
for il_tag in li_tags:
il_tag.attrs.update(il_tag.p.attrs)
il_tag.p.unwrap()
def process_html(self, access, html_path, book_id):
"""
Process html code to satisfy LiveCarta formatting.
"""
try:
self.logger_object.log(f'Processing TOC and headers.')
self._process_toc_links()
self.clean_trash()
# process main elements of the .html doc
self.logger_object.log(f'Processing main elements of html.')
self._preprocessing_headings()
self._process_paragraph()
self._process_two_columns()
self.logger_object.log('Block quotes processing.')
self._process_quotes()
self.logger_object.log('Tables processing.')
self._process_tables()
self.logger_object.log(f'{self.tables_amount} tables have been processed.')
self.logger_object.log('Hrefs processing.')
self._process_hrefs()
self.logger_object.log('Footnotes processing.')
self._process_footnotes()
self.logger_object.log(f'{len(self.footnotes)} footnotes have been processed.')
self.logger_object.log('Image processing.')
self._process_images(access=access, html_path=html_path, book_id=book_id)
self.logger_object.log(f'{len(self.images)} images have been processed.')
self._process_footer()
self._process_div()
self.content = self.body_tag.find_all(recursive=False)
self.top_level_headers = self._get_top_level_headers()
self._mark_introduction_headers()
self._process_headings()
self.content: List[Tag] = self.body_tag.find_all(recursive=False)
self._process_lists()
# delete text before table of content if exists
self.delete_content_before_toc()
except Exception as exc:
self.logger_object.log('Error has occurred while processing html.', logging.ERROR)
self.logger_object.log_error_to_main_log()
if self.status_wrapper:
self.status_wrapper.set_error()
raise exc
self.logger_object.log('End of processing .html file.')
return self.content, self.footnotes, self.top_level_headers

147
src/docx_converter/libra_html2json_converter.py Normal file
View File

@@ -0,0 +1,147 @@
import logging
import re
from copy import copy
from src.livecarta_config import LiveCartaConfig
class LibraHTML2JSONConverter:
def __init__(self, content, footnotes, top_level_headers, logger_object, book_api_status=None):
self.content_dict = None
self.content = content
self.footnotes = footnotes
self.top_level_headers = top_level_headers
self.logger_object = logger_object
self.book_api_status = book_api_status
@staticmethod
def format_html(html_text):
"""
Function to remove useless symbols from html code.
:param html_text: Text to process.
:return: Cleaned text.
"""
new_text = re.sub(r'([\n\t])', ' ', html_text)
return new_text
# TODO: rethink the function structure without indexes.
def header_to_livecarta_chapter_item(self, ind) -> (dict, int):
"""
Function process header and collects all content for it.
:param ind: Index of header in content list.
"""
if self.content[ind].name in LiveCartaConfig.SUPPORTED_HEADERS:
title = str(self.content[ind])
title = title.replace(f'<{self.content[ind].name}>', '')
title = title.replace(f'</{self.content[ind].name}>', '')
title = re.sub(r'^\n', '', title)
curr_outline = int(re.sub(r"^h", "", self.content[ind].name)) # extract outline from tag
result = {
'title': f'{title}',
'contents': [],
'sub_items': []
}
ch_content = []
ind += 1
while ind < len(self.content):
# 1. next tag is a header
if self.content[ind].name in LiveCartaConfig.SUPPORTED_HEADERS:
outline = int(re.sub(r"^h", "", self.content[ind].name))
# - recursion step until h_i > h_initial
if outline > curr_outline:
header_dict, ind = self.header_to_livecarta_chapter_item(ind)
if ch_content:
result['contents'].append("".join(ch_content))
ch_content = []
result['sub_items'].append(header_dict)
# - current h_i <= h_initial, end of recursion
else:
# return result, ind
break
# 2. next tag is not a header. add new paragraphs
else:
html_str = self.format_html(str(self.content[ind]))
ch_content.append(html_str)
ind += 1
if ch_content:
result['contents'].append("".join(ch_content))
return result, ind
return ''
@staticmethod
def _is_empty_p_tag(tag):
if tag.name != 'p':
return False
temp_tag = copy(tag)
brs = temp_tag.find_all('br')
for br in brs:
br.decompose()
text = re.sub(r'\s+', '', temp_tag.text)
if text:
return False
return True
def convert_to_dict(self):
"""
Function which convert list of html nodes to appropriate json structure.
"""
json_strc = []
ind = 0
ch_num = 0
ch_amt = 0
try:
while ind < len(self.content):
res = {}
if self.content[ind].name in LiveCartaConfig.SUPPORTED_HEADERS:
res, ind = self.header_to_livecarta_chapter_item(ind)
else:
chapter_title = f'Untitled chapter {ch_num}'
chapter = []
while ind < len(self.content) and self.content[ind].name not in LiveCartaConfig.SUPPORTED_HEADERS:
if not self._is_empty_p_tag(self.content[ind]):
chapter.append(self.format_html(str(self.content[ind])))
ind += 1
if chapter:
res = {
'title': chapter_title,
'contents': ["".join(chapter)],
'sub_items': []
}
ch_num += 1
if res:
json_strc.append(res)
ch_amt += 1
self.logger_object.log(f'Chapter {ch_amt} has been added to structure.')
except Exception as exc:
self.logger_object.log('Error has occurred while making json structure.', logging.ERROR)
self.logger_object.log_error_to_main_log()
if self.book_api_status:
self.book_api_status.set_error()
raise exc
# Add is_introduction field to json structure
# after deleting content before toc, some chapters can be deleted
if self.top_level_headers:
same_first_titles = self.top_level_headers[0]['title'] == json_strc[0]['title']
is_first_header_introduction = not self.top_level_headers[0]['should_be_numbered']
json_strc[0]['is_introduction'] = is_first_header_introduction
self.content_dict = {
"content": json_strc,
"footnotes": self.footnotes
}
return self.content_dict