lxbfYeaa/BookConverter
Archived
Template
1
0
Fork 0
forked from LiveCarta/BookConverter
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

put style processors on general level

Browse Source
This commit is contained in:
Kiryl
2022-09-01 18:12:04 +03:00
parent 39d5e27df2
commit 115a53e366
5 changed files with 35 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
src/epub_converter/css_processor.py
View File

@@ -1,220 +0,0 @@
import re
import cssutils
from bs4 import BeautifulSoup
from typing import Tuple, Dict
from os.path import dirname, normpath, join
from src.util.color_reader import str2hex
from src.livecarta_config import LiveCartaConfig
class CSSPreprocessor:
def __init__(self):
"""
Dictionary LIVECARTA_STYLE_ATTRS_MAPPING = { property: mapping function }
Warning, if LIVECARTA_STYLE_ATTRS is changed, LIVECARTA_STYLE_ATTRS_MAPPING should be updated
to suit LiveCarta style convention.
"""
self.LIVECARTA_STYLE_ATTRS_MAPPING = {
"text-indent": self.convert_indents_tag_values,
"font-variant": lambda x: x,
"text-align": lambda x: x,
"font": lambda x: "",
"font-family": lambda x: x,
"font-size": self.convert_tag_style_values,
"color": self.get_text_color,
"background-color": self.get_bg_color,
"background": self.get_bg_color,
"border": lambda x: x if x != "0" else "",
"border-top-width": lambda x: x if x != "0" else "",
"border-right-width": lambda x: x if x != "0" else "",
"border-left-width": lambda x: x if x != "0" else "",
"border-bottom-width": lambda x: x if x != "0" else "",
"border-top": lambda x: x if x != "0" else "",
"border-bottom": lambda x: x if x != "0" else "",
"list-style-type": lambda x: x if x in LiveCartaConfig.list_types else "disc",
"list-style-image": lambda x: "disc",
"margin-left": self.convert_indents_tag_values,
"margin-top": self.convert_tag_style_values,
"margin": self.convert_indents_tag_values,
"width": self.convert_tag_style_values,
}
@staticmethod
def get_text_color(x: str) -> str:
color = str2hex(x)
color = color if color not in ["#000000", "#000", "black"] else ""
return color
@staticmethod
def get_bg_color(x: str) -> str:
color = str2hex(x)
color = color if color not in ["#ffffff", "#fff", "white"] else ""
return color
@staticmethod
def convert_tag_style_values(size_value: str, is_indent: bool = False) -> str:
"""
Function
- converts values of tags from em/%/pt/in to px
- find closest font-size px
Parameters
----------
size_value: str
is_indent: bool
Returns
-------
size_value: str
converted value size
"""
size_regexp = re.compile(
r"(^-*(\d*\.*\d+)%$)|(^-*(\d*\.*\d+)em$)|(^-*(\d*\.*\d+)pt$)|(^-*(\d*\.*\d+)in$)")
has_style_attrs = re.search(size_regexp, size_value)
if has_style_attrs:
if has_style_attrs.group(1):
multiplier = 5.76 if is_indent else 0.16
size_value = float(size_value.replace("%", "")) * multiplier
return str(size_value)+'px'
elif has_style_attrs.group(3):
multiplier = 18 if is_indent else 16
size_value = float(size_value.replace("em", "")) * multiplier
return str(size_value)+'px'
elif has_style_attrs.group(5):
size_value = float(size_value.replace("pt", "")) * 4/3
return str(size_value)+'px'
elif has_style_attrs.group(7):
size_value = float(size_value.replace("in", "")) * 96
return str(size_value)+'px'
else:
return ""
return size_value
def convert_indents_tag_values(self, size_value: str) -> str:
"""
Function converts values of ["text-indent", "margin-left", "margin"]
Parameters
----------
size_value: str
Returns
-------
size_value: str
"""
size_value = self.convert_tag_style_values(size_value.split(" ")[-2], True) if len(size_value.split(" ")) == 3\
else self.convert_tag_style_values(size_value.split(" ")[-1], True)
return size_value
@staticmethod
def clean_value(style_value: str, style_name: str):
cleaned_value = style_value.replace("\"", "")
if style_name == 'font-family':
for symbol in ["+", "*", ".", "%", "?", "$", "^", "[", "]"]:
cleaned_value = re.sub(
re.escape(f"{symbol}"), rf"\\{symbol}", cleaned_value)
return cleaned_value
@staticmethod
def style_conditions(style_value: str, style_name: str) -> Tuple[bool, bool]:
constraints_on_value = LiveCartaConfig.LIVECARTA_STYLE_ATTRS.get(
style_name)
value_not_in_possible_values_list = style_value not in LiveCartaConfig.LIVECARTA_STYLE_ATTRS[
style_name]
return constraints_on_value, value_not_in_possible_values_list
def update_inline_styles_to_livecarta_convention(self, split_style: list) -> list:
for i, style in enumerate(split_style):
style_name, style_value = style.split(":")
if style_name not in LiveCartaConfig.LIVECARTA_STYLE_ATTRS:
# property not in LIVECARTA_STYLE_ATTRS, remove from css file
split_style[i] = ""
return split_style
cleaned_value = self.clean_value(style_value, style_name)
if all(self.style_conditions(cleaned_value, style_name)):
# there are constraints + value not in LIVECARTA_STYLE_ATTRS, remove from css file
split_style[i] = ""
else:
if style_name in self.LIVECARTA_STYLE_ATTRS_MAPPING:
# function that converts our data
func = self.LIVECARTA_STYLE_ATTRS_MAPPING[style_name]
style_value = func(cleaned_value)
split_style[i] = style_name + ":" + style_value
return split_style
def build_inline_style_content(self, style: str) -> str:
"""Build inline style with LiveCarta convention"""
# replace all spaces between "; & letter" to ";"
style = re.sub(r"; *", ";", style)
# when we split style by ";", last element of the list is "" - None (we remove it)
split_style: list = list(filter(None, style.split(";")))
# replace all spaces between ": & letter" to ":"
split_style = [el.replace(
re.search(r"(:\s*)", el).group(1), ":") for el in split_style]
split_style = self.update_inline_styles_to_livecarta_convention(
split_style)
style = "; ".join(split_style)
return style
def process_inline_styles_in_html_soup(self, html_href2html_body_soup: Dict[str, BeautifulSoup]):
"""This function is designed to convert inline html styles"""
for html_href in html_href2html_body_soup:
html_content: BeautifulSoup = html_href2html_body_soup[html_href]
tags_with_inline_style = html_content.find_all(LiveCartaConfig.could_have_style_in_livecarta_regexp,
attrs={"style": re.compile(".*")})
for tag_initial_inline_style in tags_with_inline_style:
inline_style = tag_initial_inline_style.attrs["style"]
tag_initial_inline_style.attrs["style"] = \
self.build_inline_style_content(inline_style)
@staticmethod
def get_css_content(css_href: str, html_href: str, ebooklib_book) -> str:
path_to_css_from_html = css_href
html_folder = dirname(html_href)
path_to_css_from_root = normpath(
join(html_folder, path_to_css_from_html)).replace("\\", "/")
css_obj = ebooklib_book.get_item_with_href(path_to_css_from_root)
# if in css file we import another css
if "@import" in str(css_obj.content):
path_to_css_from_root = "css/" + \
re.search('"(.*)"', str(css_obj.content)).group(1)
css_obj = ebooklib_book.get_item_with_href(
path_to_css_from_root)
assert css_obj, f"Css style {css_href} was not in manifest."
css_content: str = css_obj.get_content().decode()
return css_content
def update_css_styles_to_livecarta_convention(self, css_rule: cssutils.css.CSSStyleRule,
style_type: cssutils.css.property.Property):
if style_type.name not in LiveCartaConfig.LIVECARTA_STYLE_ATTRS:
# property not in LIVECARTA_STYLE_ATTRS, remove from css file
css_rule.style[style_type.name] = ""
return
cleaned_value = self.clean_value(style_type.value, style_type.name)
if all(self.style_conditions(cleaned_value, style_type.name)):
# there are constraints + value not in LIVECARTA_STYLE_ATTRS, remove from css file
css_rule.style[style_type.name] = ""
else:
if style_type.name in self.LIVECARTA_STYLE_ATTRS_MAPPING:
# function that converts our data
func = self.LIVECARTA_STYLE_ATTRS_MAPPING[style_type.name]
css_rule.style[style_type.name] = func(cleaned_value)
def build_css_file_content(self, css_content: str) -> str:
"""Build css content with LiveCarta convention"""
sheet = cssutils.parseString(css_content, validate=False)
for css_rule in sheet:
if css_rule.type == css_rule.STYLE_RULE:
for style_type in css_rule.style:
self.update_css_styles_to_livecarta_convention(
css_rule, style_type)
css_text: str = sheet._getCssText().decode()
return css_text

12
src/epub_converter/epub_converter.py
View File

@@ -15,15 +15,15 @@ from bs4 import BeautifulSoup, Tag, NavigableString
from src.util.helpers import BookLogger
from src.livecarta_config import LiveCartaConfig
from src.data_objects import ChapterItem, NavPoint
from src.epub_converter.css_processor import CSSPreprocessor
from src.style_preprocessor import CSSPreprocessor
from src.epub_converter.html_epub_processor import HtmlEpubPreprocessor
from src.epub_converter.image_processing import update_images_src_links
from src.epub_converter.footnotes_processing import preprocess_footnotes
from src.epub_converter.tag_inline_style_processor import TagInlineStyleProcessor
from src.tag_inline_style_processor import TagInlineStyleProcessor
class EpubConverter:
def __init__(self, book_path, access=None, logger=None, css_processor=None, html_processor=None):
def __init__(self, book_path, access=None, logger: BookLogger = None, css_processor: CSSPreprocessor = None, html_processor: HtmlEpubPreprocessor = None):
self.book_path = book_path
self.access = access
self.logger: BookLogger = logger
@@ -257,7 +257,7 @@ class EpubConverter:
sub_nodes = []
for elem in second:
if (bool(re.search('^section$|^part$', first.title.lower()))) and lvl == 1:
if (bool(re.search("^section$|^part$", first.title.lower()))) and lvl == 1:
self.offset_sub_nodes.append(
self.build_adjacency_list_from_toc(elem, lvl))
else:
@@ -291,7 +291,7 @@ class EpubConverter:
return False
def build_adjacency_list_from_spine(self):
def build_manifest_id2html_href() -> dict:
def build_manifest_id2html_href() -> Dict[int, str]:
links = dict()
for item in self.ebooklib_book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
links[item.id] = item.file_name
@@ -607,7 +607,7 @@ class EpubConverter:
self.logger.log(indent + "Process title.")
title_preprocessed: str = self.html_processor.prepare_title(title)
self.logger.log(indent + "Process content.")
content_preprocessed: BeautifulSoup = self.html_processor.prepare_content(
content_preprocessed: Union[Tag, BeautifulSoup] = self.html_processor.prepare_content(
title_preprocessed, content, remove_title_from_chapter=is_chapter)
self.book_image_src_path2aws_path = update_images_src_links(content_preprocessed,

2
src/epub_converter/epub_solver.py
View File

@@ -1,5 +1,5 @@
from src.book_solver import BookSolver
from src.epub_converter.css_processor import CSSPreprocessor
from src.style_preprocessor import CSSPreprocessor
from src.epub_converter.html_epub_processor import HtmlEpubPreprocessor
from src.epub_converter.epub_converter import EpubConverter

47
src/epub_converter/html_epub_processor.py
View File

@@ -192,14 +192,18 @@ class HtmlEpubPreprocessor:
tag_to_replace: str = rule["tag_to_replace"]
if rule["condition"]:
for condition_on_tag in ((k, v) for k, v in rule["condition"].items() if v):
if condition_on_tag[0] == 'parent_tags':
if condition_on_tag[0] == "parent_tags":
for tag in chapter_tag.find_all([re.compile(tag) for tag in tags]):
if tag.parent.select(condition_on_tag[1]):
tag.name = tag_to_replace
elif condition_on_tag[0] == 'child_tags':
elif condition_on_tag[0] == "child_tags":
for tag in chapter_tag.find_all([re.compile(tag) for tag in tags]):
if not tag.select(re.sub('[():]|not', '', condition_on_tag[1])):
tag.name = tag_to_replace
if "not" in condition_on_tag[1]:
if not tag.select(re.sub("[():]|not", "", condition_on_tag[1])):
tag.name = tag_to_replace
else:
if tag.select(condition_on_tag[1]):
tag.name = tag_to_replace
elif condition_on_tag[0] == "attrs":
for attr in rule["condition"]["attrs"]:
for tag in chapter_tag.find_all([re.compile(tag) for tag in tags],
@@ -236,15 +240,15 @@ class HtmlEpubPreprocessor:
tag[attr_to_replace] = tag[attr]
del tag[attr]
def _unwrap_tags(self, chapter_tag: BeautifulSoup, rules: Dict[str, List[str]]):
def _unwrap_tags(self, chapter_tag: BeautifulSoup, rules: List[Dict[str, List[str]]]):
"""
Function unwrap tags and moves id to span
Parameters
----------
chapter_tag: BeautifulSoup
Tag & contents of the chapter tag
rules: Dict[str, List[str]]
dict of tags to unwrap
rules: List[Dict[str, List[str]]]
list of conditions when fire function
Returns
-------
@@ -252,13 +256,14 @@ class HtmlEpubPreprocessor:
Chapter Tag with unwrapped certain tags
"""
for tag_name in rules["tags"]:
for tag in chapter_tag.select(tag_name):
# if tag is a subtag
if ">" in tag_name:
tag.parent.attrs.update(tag.attrs)
self._add_span_to_save_ids_for_links(tag, chapter_tag)
tag.unwrap()
for rule in rules:
for tag_name in rule["tags"]:
for tag in chapter_tag.select(tag_name):
# if tag is a subtag
if ">" in tag_name:
tag.parent.attrs.update(tag.attrs)
self._add_span_to_save_ids_for_links(tag, chapter_tag)
tag.unwrap()
@staticmethod
def _insert_tags_into_correspond_tags(chapter_tag: BeautifulSoup,
@@ -293,14 +298,18 @@ class HtmlEpubPreprocessor:
tags: List[str] = rule["tags"]
if rule["condition"]:
for condition_on_tag in ((k, v) for k, v in rule["condition"].items() if v):
if condition_on_tag[0] == 'parent_tags':
if condition_on_tag[0] == "parent_tags":
for tag in chapter_tag.find_all([re.compile(tag) for tag in tags]):
if tag.parent.select(condition_on_tag[1]):
insert(tag)
elif condition_on_tag[0] == 'child_tags':
elif condition_on_tag[0] == "child_tags":
for tag in chapter_tag.find_all([re.compile(tag) for tag in tags]):
if not tag.select(re.sub('[():]|not', '', condition_on_tag[1])):
insert(tag)
if "not" in condition_on_tag[1]:
if not tag.select(re.sub("[():]|not", "", condition_on_tag[1])):
tag.unwrap()
else:
if tag.select(condition_on_tag[1]):
tag.unwrap()
elif condition_on_tag[0] == "attrs":
for attr in rule["condition"]["attrs"]:
for tag in chapter_tag.find_all([re.compile(tag) for tag in tags],
@@ -441,7 +450,7 @@ class HtmlEpubPreprocessor:
# 3-6.
for rule in self.preset:
func = self.name2function[rule["preset_name"]]
func(content_tag, rule['rules'])
func(content_tag, rule["rules"])
# 7.
if remove_title_from_chapter:
self._remove_headings_content(content_tag, title_str)

217
src/epub_converter/tag_inline_style_processor.py
View File

@@ -1,217 +0,0 @@
import re
import cssutils
from typing import List
from logging import CRITICAL
from bs4 import BeautifulSoup, Tag
from src.livecarta_config import LiveCartaConfig
cssutils.log.setLevel(CRITICAL)
class TagInlineStyleProcessor:
def __init__(self, tag_inline_style: Tag):
# tag with inline style + style parsed from css file
self.tag_inline_style = tag_inline_style
self.tag_inline_style.attrs['style']: str = self.process_inline_style()
@staticmethod
def remove_white_if_no_bgcolor(style_: str, tag: Tag) -> str:
"""Function remove text white color if there is no bg color"""
if "background" in style_:
style_ = style_.replace(
"background:", "background-color:")
return style_
# if text color is white, check that we have bg-color
if ("color:#ffffff" in style_) or ("color:#fff" in style_) or ("color:white" in style_):
# if bg color is inherited, just return style as is
for parent_tag in tag.parents:
# white bg color not need to be checked as we do not write "white bg color"
tag_with_bg = ["span", "td", "tr", "p"]
tag_will_be_saved = parent_tag.name in tag_with_bg
has_bg = parent_tag.attrs.get("style") and (
"background" in parent_tag.attrs.get("style"))
if has_bg and tag_will_be_saved:
return style_
children = tag.find_all()
for child in children:
if child.attrs.get("style") and ("background" in child.attrs.get("style")):
tmp_style = child.attrs["style"] + "; color:#fff; "
child.attrs["style"] = tmp_style
# for child with bg color we added white text color, so this tag don"t need white color
style_ = style_.replace("color:#fff;", "")
style_ = style_.replace("color:#ffffff;", "")
style_ = style_.replace("color:white;", "")
return style_
# @staticmethod
# def duplicate_styles_check(split_style: list) -> list:
# style_name2style_value = {}
# # {key: val for for list_item in split_style}
# splitstrs = (list_item.split(":") for list_item in split_style)
# d = {key: val for key, val in splitstrs}
# for list_item in split_style:
# key, val = list_item.split(":")
# if key not in style_name2style_value.keys():
# style_name2style_value[key] = val
# split_style = [k + ":" + v for k, v in style_name2style_value.items()]
# return split_style
@staticmethod
def indents_processing(split_style: List[str]) -> str:
"""
Function process indents from left using
formula_of_indent: indent = abs(margin - text_indent)
Parameters
----------
split_style: List[str]
list of styles split by ";"
Returns
----------
processed_style:str
processed style with counted indent
"""
processed_style = ";".join(split_style)+';'
margin_left_regexp = re.compile(
r"((margin-left|margin): *(-*\w+);*)")
text_indent_regexp = re.compile(
r"(text-indent: *(-*\w+);*)")
has_margin = re.search(margin_left_regexp, processed_style)
has_text_indent = re.search(text_indent_regexp, processed_style)
if has_margin:
num_m = abs(int("0" + "".join(
filter(str.isdigit, str(has_margin.group(3))))))
if has_text_indent:
num_ti = abs(int("0" + "".join(
filter(str.isdigit, str(has_text_indent.group(2))))))
processed_style = processed_style.replace(has_text_indent.group(1), "text-indent: " +
str(abs(num_m - num_ti)) + "px; ")
processed_style = processed_style.replace(
has_margin.group(1), "")
return processed_style
processed_style = processed_style.replace(has_margin.group(1), "text-indent: " +
str(abs(num_m)) + "px; ")
return processed_style
elif has_text_indent:
processed_style = processed_style.replace(has_text_indent.group(1), "text-indent: " +
str(abs(int("0" + "".join(
filter(str.isdigit, str(has_text_indent.group(2)))))))
+ "px; ")
return processed_style
return processed_style
def process_inline_style(self) -> str:
"""
Function processes final(css+initial inline) inline style
Steps
----------
1. Remove white color if tag doesn't have background color in style
2. Create list of styles from inline style
3. Duplicate styles check - if the tag had duplicate styles
4. Processing indents
Returns
-------
inline_style: str
processed inline style
"""
inline_style = self.tag_inline_style.attrs.get("style") + ";"
# 1. Remove white color if tag doesn"t have background color in style
inline_style = self.remove_white_if_no_bgcolor(
inline_style, self.tag_inline_style)
inline_style = inline_style.replace(
"list-style-image", "list-style-type")
# 2. Create list of styles from inline style
# replace all spaces between "; & letter" to ";"
style = re.sub(r"; *", ";", inline_style)
# when we split style by ";", last element of the list is "" - None (remove it)
split_inline_style: list = list(filter(None, style.split(";")))
# 3. Duplicate styles check - if the tag had duplicate styles
# split_inline_style = self.duplicate_styles_check(split_inline_style)
# 4. Processing indents
inline_style: str = self.indents_processing(split_inline_style)
return inline_style
@staticmethod
def check_style_to_be_tag(style: str) -> List[tuple]:
"""
Function searches style properties that can be converted to tag.
It searches for them and prepare list of properties to be removed from style string
Parameters
----------
style: str
<tag style="...">
Returns
-------
styles_to_remove: list
properties to remove
"""
styles_to_remove = []
for k in LiveCartaConfig.LIVECARTA_STYLE_ATTRS_SHOULD_BE_TAG:
if f"{k[0]}:{k[1]}" in style:
styles_to_remove.append(k)
return styles_to_remove
def change_attrs_with_corresponding_tags(self):
# adds <strong>, <u>, <sup> instead of styles
styles_to_remove = self.check_style_to_be_tag(self.tag_inline_style.attrs['style'])
for i, (attr, value) in enumerate(styles_to_remove):
self.tag_inline_style.attrs["style"] = self.tag_inline_style.attrs["style"]\
.replace(f"{attr}:{value};", "").strip()
corr_tag_name = LiveCartaConfig.LIVECARTA_STYLE_ATTRS_SHOULD_BE_TAG[(
attr, value)]
correspond_tag = BeautifulSoup(features="lxml").new_tag(corr_tag_name)
for content in reversed(self.tag_inline_style.contents):
correspond_tag.insert(0, content.extract())
self.tag_inline_style.append(correspond_tag)
@staticmethod
def wrap_span_in_tag_to_save_style_attrs(initial_tag: Tag):
"""Function designed to save style attrs that cannot be in tag.name -> span"""
dictkeys_pattern = re.compile("|".join(LiveCartaConfig.LIVECARTA_STYLES_CAN_BE_IN_TAG))
if re.findall(dictkeys_pattern, initial_tag.name) and initial_tag.attrs.get("style"):
styles_can_be_in_tag = [style
for tag, styles in LiveCartaConfig.LIVECARTA_STYLES_CAN_BE_IN_TAG.items()
if re.match(tag, initial_tag.name)
for style in styles]
styles_cant_be_in_tag = [attr for attr in LiveCartaConfig.LIVECARTA_STYLE_ATTRS
if attr not in styles_can_be_in_tag]
span_style = initial_tag.attrs["style"]
# here check that this style is exactly the same.
# Not "align" when we have "text-align", or "border" when we have "border-top"
styles_to_be_saved_in_span = [((attr + ":") in span_style) & (
"-" + attr not in span_style) for attr in styles_cant_be_in_tag]
if any(styles_to_be_saved_in_span):
# if we find styles that cannot be in <tag.name> -> wrap them in span
tag = BeautifulSoup(features="lxml").new_tag(f"{initial_tag.name}")
style = ""
possible_attrs_regexp = [re.compile(fr"({style}: *\w+;)") for style in styles_can_be_in_tag]
for possible_attr_regexp in possible_attrs_regexp:
has_style_attrs = re.search(
possible_attr_regexp, span_style)
if has_style_attrs and has_style_attrs.group(1):
style += has_style_attrs.group(1)
span_style = span_style.replace(
has_style_attrs.group(1), "")
tag.attrs["style"] = style
initial_tag.name = "span"
initial_tag.attrs["style"] = span_style
initial_tag.wrap(tag)
def convert_initial_tag(self) -> Tag:
self.change_attrs_with_corresponding_tags()
self.wrap_span_in_tag_to_save_style_attrs(self.tag_inline_style)
return self.tag_inline_style