lxbfYeaa/BookConverter
Archived
Template
1
0
Fork 0
forked from LiveCarta/BookConverter
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add decomposer & unwrapper processes

Browse Source
This commit is contained in:
Kiryl
2022-09-02 14:48:01 +03:00
parent dfdf6bc7e9
commit 142c9bbe69
1 changed files with 74 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
src/docx_converter/html_docx_processor.py
View File

@@ -11,7 +11,7 @@ from src.docx_converter.footnotes_processing import process_footnotes
from src.tag_inline_style_processor import modify_html_soup_with_css_styles
class HTMLDocxPreprocessor:
class HTMLDocxProcessor:
def __init__(self, html_soup: BeautifulSoup, logger: BookLogger,
style_processor, preset_path: str = "presets/docx_presets.json"):
@@ -28,6 +28,7 @@ class HTMLDocxPreprocessor:
self.content = list()
def _process_toc_links(self):
"""Function to extract nodes which contains TOC links, remove links from file and detect headers."""
def _check_parent_link_exist_in_toc(tag_with_link: Tag) -> bool:
toc_links = []
for a_tag in tag_with_link.find_all("a", {"name": re.compile(r"^_Toc\d+")}):
@@ -36,65 +37,76 @@ class HTMLDocxPreprocessor:
if toc_item:
toc_links.append(toc_item)
return len(toc_links) > 0
"""Function to extract nodes which contains TOC links, remove links from file and detect headers."""
toc_links = self.body_tag.find_all(
"a", {"name": re.compile(r"^_Toc\d+")})
headers = [link.parent for link in toc_links]
outline_level = "1" # All the unknown outlines will be predicted as <h1>
for h_tag in headers:
if re.search(r"^h\d$", h_tag.name):
h_tag.a.unwrap()
# outline_level = tag.name[-1] # TODO: add prediction of the outline level
elif h_tag.name == "p":
exist_in_toc = _check_parent_link_exist_in_toc(h_tag)
if h_tag in self.body_tag.find_all("p") and exist_in_toc:
for tag in headers:
if re.search(r"^h\d$", tag.name):
tag.a.unwrap()
elif tag.name == "p":
exist_in_toc = _check_parent_link_exist_in_toc(tag)
if tag in self.body_tag.find_all("p") and exist_in_toc:
new_tag = BeautifulSoup(
features="lxml").new_tag("h" + outline_level)
text = h_tag.text
h_tag.replaceWith(new_tag)
text = tag.text
tag.replaceWith(new_tag)
new_tag.string = text
else:
# rethink document structure when you have toc_links, other cases?
self.logger_object.log(f"Something went wrong in processing toc_links."
f" Check the structure of the file. "
f"Tag name: {h_tag.name}")
self.logger.log(f"Something went wrong in processing toc_links."
f"Check the structure of the file."
f"Tag name: {tag.name}")
def _clean_tag(self, tag: str, attr_name: str, attr_value: re):
# todo regex
@staticmethod
def _decompose_tag(tag):
tag.decompose()
@staticmethod
def _unwrap_tag(tag):
tag.unwrap()
@staticmethod
def _process_tags(body_tag: BeautifulSoup,
rules: List[Dict[str, Union[List[str], str, Dict[str, Union[List[Dict[str, str]], int, str]]]]],
action):
"""
Function to clean tags by its name and attribute value.
Function do action with tags
Parameters
----------
tag: str
tag name to clean
attr_name: str
attribute name
attr_value: [str,re]
attribute value
body_tag: BeautifulSoup
Tag & contents of the chapter tag
rules: List[Dict[str, Union[List[str], str, Dict[str, Union[List[Dict[str, str]], int, str]]]]]
list of conditions when fire function
action: function
action what to do with tag
Returns
-------
clean tag
NoReturn
Body Tag with processed certain tags
"""
tags = self.body_tag.find_all(tag, {attr_name: attr_value})
for tag in tags:
if len(tag.attrs) == 1:
tag.unwrap()
def _clean_underline_links(self):
# todo regex
"""Function cleans meaningless <u> tags before links."""
underlines = self.body_tag.find_all("u")
for u in underlines:
if u.find_all("a"):
u.unwrap()
links = self.body_tag.find_all("a")
for link in links:
u = link.find_all("u")
if u and len(u) == 1:
u[0].unwrap()
for rule in rules:
tags: List[str] = rule["tags"]
if rule["condition"]:
for condition_on_tag in ((k, v) for k, v in rule["condition"].items() if v):
if condition_on_tag[0] == "parent_tags":
for tag in body_tag.select(', '.join([condition_on_tag[1] + " > " + re.sub(r"[\^$]", "", tag)
for tag in tags])):
tag.parent.attrs.update(tag.attrs)
action(tag)
elif condition_on_tag[0] == "child_tags":
for tag in body_tag.select(', '.join([re.sub(r"[\^$]", "", tag) + condition_on_tag[1]
for tag in tags])):
action(tag)
elif condition_on_tag[0] == "attrs":
for attr in rule["condition"]["attrs"]:
for tag in body_tag.find_all([re.compile(tag) for tag in tags],
{attr["name"]: re.compile(fr"{attr['value']}")}):
action(tag)
else:
for tag in body_tag.find_all([re.compile(tag) for tag in tags]):
action(tag)
@classmethod
def convert_pt_to_px(cls, value: float) -> float:
@@ -155,21 +167,8 @@ class HTMLDocxPreprocessor:
assert len(self.body_tag.find_all("font")) == 0
def clean_trash(self):
# todo make it regex dict
"""Function to remove all styles and tags we don"t need."""
self._clean_tag("span", "style", re.compile(
r"^background: #[\da-fA-F]{6}$"))
# todo: check for another languages
self._clean_tag("span", "lang", re.compile(r"^ru-RU$"))
self._clean_tag("span", "style", re.compile(
"^letter-spacing: -?[\d.]+pt$"))
self._clean_tag("font", "face", re.compile(
r"^Times New Roman[\w, ]+$"))
self._clean_tag("a", "name", "_GoBack")
self._clean_underline_links()
# todo replacer
self._font_to_span()
# replace toc with empty <TOC> tag
@@ -180,7 +179,7 @@ class HTMLDocxPreprocessor:
table.decompose()
def _preprocessing_headings(self):
# todo regex
# todo replacer
"""Function to convert all lower level headings to p tags"""
pattern = f"^h[{LiveCartaConfig.SUPPORTED_LEVELS + 1}-9]$"
header_tags = self.body_tag.find_all(re.compile(pattern))
@@ -189,6 +188,7 @@ class HTMLDocxPreprocessor:
def _process_paragraph(self):
"""Function to process <p> tags (text-align and text-indent value)."""
# todo debug and remove if inline is enough
paragraphs = self.body_tag.find_all("p")
for p in paragraphs:
@@ -239,6 +239,7 @@ class HTMLDocxPreprocessor:
def _process_two_columns(self):
"""Function to process paragraphs which has two columns layout."""
# todo replacer
two_columns = self.body_tag.find_all("div", style="column-count: 2")
for div in two_columns:
for child in div.children:
@@ -289,14 +290,11 @@ class HTMLDocxPreprocessor:
tables = self.body_tag.find_all("table")
for table in tables:
tds = table.find_all("td")
sizes = []
for td in tds:
style = td.get("style")
if style:
match = re.search(r"border: ?(\d+\.?\d*)(p[tx])", style)
if match:
size = match.group(1)
units = match.group(2)
@@ -305,13 +303,10 @@ class HTMLDocxPreprocessor:
size = self.convert_pt_to_px(size)
sizes.append(float(size))
width = td.get("width")
td.attrs = {}
if width:
td.attrs["width"] = width
if sizes:
border_size = sum(sizes) / len(sizes)
table.attrs["border"] = f"{border_size:.2}"
@@ -334,18 +329,8 @@ class HTMLDocxPreprocessor:
tag.string = tag.text.replace("\u200b", "") # zero-width-space
tag["href"] = tag.attrs.get("href").replace("%E2%80%8C", "")
def _process_footer(self):
# todo regex
"""
Function to process <div title="footer"> tags.
All the tags will be deleted from file.
"""
divs = self.body_tag.find_all("div", {"title": "footer"})
for div in divs:
div.decompose()
def _process_div(self):
# todo regex
# todo unwrapper
"""Function to process <div> tags. All the tags will be deleted from file, all content of the tags will stay."""
divs = self.body_tag.find_all("div")
for div in divs:
@@ -394,7 +379,7 @@ class HTMLDocxPreprocessor:
def _mark_introduction_headers(self):
"""
Function to find out:
what header shouldn"t be numbered and can be treated as introduction chapter
what header shouldn't be numbered and can be treated as introduction chapter
Assume header(s) to be introduction if:
1. one header not numbered, before 1 numbered header
2. it is first header from the top level list, and it equals to "introduction"
@@ -442,13 +427,9 @@ class HTMLDocxPreprocessor:
self.apply_func_to_last_child(children[0], func)
def _process_headings(self):
# todo regex
"""
Function to process tags <h>.
Steps
----------
1. remove <b>, <span>
2. clean text in header from numbering and \n
Clean header from attrs and text in header from numbering and \n
Returns
-------
@@ -458,34 +439,22 @@ class HTMLDocxPreprocessor:
"""
header_tags = self.body_tag.find_all(re.compile("^h[1-9]$"))
# 1. remove <b>, <span>
for tag in header_tags:
b_tags = tag.find_all("b")
[tag.unwrap() for tag in b_tags]
# clean header from attrs and text in header from numbering and \n
for h_tag in header_tags:
h_tag.attrs = {}
if h_tag.parent.name == "li":
h_tag.parent.unwrap()
while h_tag.parent.name == "ol":
h_tag.parent.unwrap()
spans = tag.find_all("span")
if spans:
[span.unwrap() for span in spans]
tag.attrs = {}
header_tags = self.body_tag.find_all(re.compile("^h[1-9]$"))
# 2. clean text in header from numbering and \n
for tag in header_tags:
if tag.parent.name == "li":
tag.parent.unwrap()
while tag.parent.name == "ol":
tag.parent.unwrap()
cleaned_title = re.sub(r"[\s\xa0]", " ", tag.text)
cleaned_title = re.sub(r"[\s\xa0]", " ", h_tag.text)
if cleaned_title == "":
tag.unwrap()
h_tag.unwrap()
else:
assert tag.name in LiveCartaConfig.SUPPORTED_HEADERS, \
assert h_tag.name in LiveCartaConfig.SUPPORTED_HEADERS, \
f"Preprocessing went wrong, there is still h{LiveCartaConfig.SUPPORTED_LEVELS + 1}-h9 headings."
content = list(tag.children)
content = list(h_tag.children)
# do not take into account rubbish empty tags like <a>, but don"t remove them
content = [item for item in content if
@@ -506,24 +475,6 @@ class HTMLDocxPreprocessor:
self.apply_func_to_last_child(
content[i], self.clean_title_from_tabs)
def _process_lists(self):
# todo regex
"""
Function
- process tags <li>.
- unwrap <p> tags.
Returns
-------
None
uwrap <p> tag with li
"""
li_tags = self.body_tag.find_all("li")
for li_tag in li_tags:
li_tag.attrs.update(li_tag.p.attrs)
li_tag.p.unwrap()
def delete_content_before_toc(self):
# remove all tag upper the <TOC> only in content !!! body tag is not updated
toc_tag = self.html_soup.new_tag("TOC")
@@ -553,7 +504,7 @@ class HTMLDocxPreprocessor:
self.clean_trash()
# process main elements of the .html doc
self.logger_object.log(f"Processing main elements of html.")
self.logger.log(f"Processing main elements of html.")
self._preprocessing_headings()
self._process_paragraph()
self._process_two_columns()
@@ -587,7 +538,6 @@ class HTMLDocxPreprocessor:
self._process_headings()
self._process_lists()
# delete text before table of content if exists
self.delete_content_before_toc()