diff --git a/src/docx_converter/html_docx_processor.py b/src/docx_converter/html_docx_processor.py index 959ef55..9ea15c2 100644 --- a/src/docx_converter/html_docx_processor.py +++ b/src/docx_converter/html_docx_processor.py @@ -11,7 +11,7 @@ from src.docx_converter.footnotes_processing import process_footnotes from src.tag_inline_style_processor import modify_html_soup_with_css_styles -class HTMLDocxPreprocessor: +class HTMLDocxProcessor: def __init__(self, html_soup: BeautifulSoup, logger: BookLogger, style_processor, preset_path: str = "presets/docx_presets.json"): @@ -28,6 +28,7 @@ class HTMLDocxPreprocessor: self.content = list() def _process_toc_links(self): + """Function to extract nodes which contains TOC links, remove links from file and detect headers.""" def _check_parent_link_exist_in_toc(tag_with_link: Tag) -> bool: toc_links = [] for a_tag in tag_with_link.find_all("a", {"name": re.compile(r"^_Toc\d+")}): @@ -36,65 +37,76 @@ class HTMLDocxPreprocessor: if toc_item: toc_links.append(toc_item) return len(toc_links) > 0 - """Function to extract nodes which contains TOC links, remove links from file and detect headers.""" toc_links = self.body_tag.find_all( "a", {"name": re.compile(r"^_Toc\d+")}) headers = [link.parent for link in toc_links] outline_level = "1" # All the unknown outlines will be predicted as

- for h_tag in headers: - if re.search(r"^h\d$", h_tag.name): - h_tag.a.unwrap() - # outline_level = tag.name[-1] # TODO: add prediction of the outline level - elif h_tag.name == "p": - exist_in_toc = _check_parent_link_exist_in_toc(h_tag) - if h_tag in self.body_tag.find_all("p") and exist_in_toc: + for tag in headers: + if re.search(r"^h\d$", tag.name): + tag.a.unwrap() + elif tag.name == "p": + exist_in_toc = _check_parent_link_exist_in_toc(tag) + if tag in self.body_tag.find_all("p") and exist_in_toc: new_tag = BeautifulSoup( features="lxml").new_tag("h" + outline_level) - text = h_tag.text - h_tag.replaceWith(new_tag) + text = tag.text + tag.replaceWith(new_tag) new_tag.string = text else: # rethink document structure when you have toc_links, other cases? - self.logger_object.log(f"Something went wrong in processing toc_links." - f" Check the structure of the file. " - f"Tag name: {h_tag.name}") + self.logger.log(f"Something went wrong in processing toc_links." + f"Check the structure of the file." + f"Tag name: {tag.name}") - def _clean_tag(self, tag: str, attr_name: str, attr_value: re): - # todo regex + @staticmethod + def _decompose_tag(tag): + tag.decompose() + + @staticmethod + def _unwrap_tag(tag): + tag.unwrap() + + @staticmethod + def _process_tags(body_tag: BeautifulSoup, + rules: List[Dict[str, Union[List[str], str, Dict[str, Union[List[Dict[str, str]], int, str]]]]], + action): """ - Function to clean tags by its name and attribute value. + Function do action with tags Parameters ---------- - tag: str - tag name to clean - attr_name: str - attribute name - attr_value: [str,re] - attribute value - + body_tag: BeautifulSoup + Tag & contents of the chapter tag + rules: List[Dict[str, Union[List[str], str, Dict[str, Union[List[Dict[str, str]], int, str]]]]] + list of conditions when fire function + action: function + action what to do with tag Returns ------- - clean tag + NoReturn + Body Tag with processed certain tags """ - tags = self.body_tag.find_all(tag, {attr_name: attr_value}) - for tag in tags: - if len(tag.attrs) == 1: - tag.unwrap() - - def _clean_underline_links(self): - # todo regex - """Function cleans meaningless tags before links.""" - underlines = self.body_tag.find_all("u") - for u in underlines: - if u.find_all("a"): - u.unwrap() - - links = self.body_tag.find_all("a") - for link in links: - u = link.find_all("u") - if u and len(u) == 1: - u[0].unwrap() + for rule in rules: + tags: List[str] = rule["tags"] + if rule["condition"]: + for condition_on_tag in ((k, v) for k, v in rule["condition"].items() if v): + if condition_on_tag[0] == "parent_tags": + for tag in body_tag.select(', '.join([condition_on_tag[1] + " > " + re.sub(r"[\^$]", "", tag) + for tag in tags])): + tag.parent.attrs.update(tag.attrs) + action(tag) + elif condition_on_tag[0] == "child_tags": + for tag in body_tag.select(', '.join([re.sub(r"[\^$]", "", tag) + condition_on_tag[1] + for tag in tags])): + action(tag) + elif condition_on_tag[0] == "attrs": + for attr in rule["condition"]["attrs"]: + for tag in body_tag.find_all([re.compile(tag) for tag in tags], + {attr["name"]: re.compile(fr"{attr['value']}")}): + action(tag) + else: + for tag in body_tag.find_all([re.compile(tag) for tag in tags]): + action(tag) @classmethod def convert_pt_to_px(cls, value: float) -> float: @@ -155,21 +167,8 @@ class HTMLDocxPreprocessor: assert len(self.body_tag.find_all("font")) == 0 def clean_trash(self): - # todo make it regex dict """Function to remove all styles and tags we don"t need.""" - self._clean_tag("span", "style", re.compile( - r"^background: #[\da-fA-F]{6}$")) - # todo: check for another languages - self._clean_tag("span", "lang", re.compile(r"^ru-RU$")) - self._clean_tag("span", "style", re.compile( - "^letter-spacing: -?[\d.]+pt$")) - - self._clean_tag("font", "face", re.compile( - r"^Times New Roman[\w, ]+$")) - - self._clean_tag("a", "name", "_GoBack") - self._clean_underline_links() - + # todo replacer self._font_to_span() # replace toc with empty tag @@ -180,7 +179,7 @@ class HTMLDocxPreprocessor: table.decompose() def _preprocessing_headings(self): - # todo regex + # todo replacer """Function to convert all lower level headings to p tags""" pattern = f"^h[{LiveCartaConfig.SUPPORTED_LEVELS + 1}-9]$" header_tags = self.body_tag.find_all(re.compile(pattern)) @@ -189,6 +188,7 @@ class HTMLDocxPreprocessor: def _process_paragraph(self): """Function to process

tags (text-align and text-indent value).""" + # todo debug and remove if inline is enough paragraphs = self.body_tag.find_all("p") for p in paragraphs: @@ -239,6 +239,7 @@ class HTMLDocxPreprocessor: def _process_two_columns(self): """Function to process paragraphs which has two columns layout.""" + # todo replacer two_columns = self.body_tag.find_all("div", style="column-count: 2") for div in two_columns: for child in div.children: @@ -289,14 +290,11 @@ class HTMLDocxPreprocessor: tables = self.body_tag.find_all("table") for table in tables: tds = table.find_all("td") - sizes = [] for td in tds: style = td.get("style") - if style: match = re.search(r"border: ?(\d+\.?\d*)(p[tx])", style) - if match: size = match.group(1) units = match.group(2) @@ -305,13 +303,10 @@ class HTMLDocxPreprocessor: size = self.convert_pt_to_px(size) sizes.append(float(size)) - width = td.get("width") - td.attrs = {} if width: td.attrs["width"] = width - if sizes: border_size = sum(sizes) / len(sizes) table.attrs["border"] = f"{border_size:.2}" @@ -334,18 +329,8 @@ class HTMLDocxPreprocessor: tag.string = tag.text.replace("\u200b", "") # zero-width-space tag["href"] = tag.attrs.get("href").replace("%E2%80%8C", "") - def _process_footer(self): - # todo regex - """ - Function to process

tags. - All the tags will be deleted from file. - """ - divs = self.body_tag.find_all("div", {"title": "footer"}) - for div in divs: - div.decompose() - def _process_div(self): - # todo regex + # todo unwrapper """Function to process
tags. All the tags will be deleted from file, all content of the tags will stay.""" divs = self.body_tag.find_all("div") for div in divs: @@ -394,7 +379,7 @@ class HTMLDocxPreprocessor: def _mark_introduction_headers(self): """ Function to find out: - what header shouldn"t be numbered and can be treated as introduction chapter + what header shouldn't be numbered and can be treated as introduction chapter Assume header(s) to be introduction if: 1. one header not numbered, before 1 numbered header 2. it is first header from the top level list, and it equals to "introduction" @@ -442,13 +427,9 @@ class HTMLDocxPreprocessor: self.apply_func_to_last_child(children[0], func) def _process_headings(self): - # todo regex """ Function to process tags . - Steps - ---------- - 1. remove , - 2. clean text in header from numbering and \n + Clean header from attrs and text in header from numbering and \n Returns ------- @@ -458,34 +439,22 @@ class HTMLDocxPreprocessor: """ header_tags = self.body_tag.find_all(re.compile("^h[1-9]$")) - # 1. remove , - for tag in header_tags: - b_tags = tag.find_all("b") - [tag.unwrap() for tag in b_tags] + # clean header from attrs and text in header from numbering and \n + for h_tag in header_tags: + h_tag.attrs = {} + if h_tag.parent.name == "li": + h_tag.parent.unwrap() + while h_tag.parent.name == "ol": + h_tag.parent.unwrap() - spans = tag.find_all("span") - - if spans: - [span.unwrap() for span in spans] - tag.attrs = {} - - header_tags = self.body_tag.find_all(re.compile("^h[1-9]$")) - - # 2. clean text in header from numbering and \n - for tag in header_tags: - if tag.parent.name == "li": - tag.parent.unwrap() - while tag.parent.name == "ol": - tag.parent.unwrap() - - cleaned_title = re.sub(r"[\s\xa0]", " ", tag.text) + cleaned_title = re.sub(r"[\s\xa0]", " ", h_tag.text) if cleaned_title == "": - tag.unwrap() + h_tag.unwrap() else: - assert tag.name in LiveCartaConfig.SUPPORTED_HEADERS, \ + assert h_tag.name in LiveCartaConfig.SUPPORTED_HEADERS, \ f"Preprocessing went wrong, there is still h{LiveCartaConfig.SUPPORTED_LEVELS + 1}-h9 headings." - content = list(tag.children) + content = list(h_tag.children) # do not take into account rubbish empty tags like , but don"t remove them content = [item for item in content if @@ -506,24 +475,6 @@ class HTMLDocxPreprocessor: self.apply_func_to_last_child( content[i], self.clean_title_from_tabs) - def _process_lists(self): - # todo regex - """ - Function - - process tags
  • . - - unwrap

    tags. - - Returns - ------- - None - uwrap

    tag with li - - """ - li_tags = self.body_tag.find_all("li") - for li_tag in li_tags: - li_tag.attrs.update(li_tag.p.attrs) - li_tag.p.unwrap() - def delete_content_before_toc(self): # remove all tag upper the only in content !!! body tag is not updated toc_tag = self.html_soup.new_tag("TOC") @@ -553,7 +504,7 @@ class HTMLDocxPreprocessor: self.clean_trash() # process main elements of the .html doc - self.logger_object.log(f"Processing main elements of html.") + self.logger.log(f"Processing main elements of html.") self._preprocessing_headings() self._process_paragraph() self._process_two_columns() @@ -587,7 +538,6 @@ class HTMLDocxPreprocessor: self._process_headings() - self._process_lists() # delete text before table of content if exists self.delete_content_before_toc()