Coverage for meta_tags_parser/parse.py: 100%
110 statements
« prev ^ index » next coverage.py v7.10.3, created at 2026-08-22 19:08 +0000
1import codecs
2import contextvars
3import functools
4import re
5import typing
6from collections.abc import KeysView
8from selectolax.lexbor import LexborHTMLParser, LexborNode
10from . import structs
13_GLOBAL_OPTIONS_HOLDER: typing.Final[contextvars.ContextVar[structs.SettingsFromUser]] = contextvars.ContextVar(
14 "options", default=structs.DEFAULT_SETTINGS_FROM_USER
15)
16_CHARSET_DECLARATION_PATTERN: typing.Final = re.compile(rb"""charset\s*=\s*["']?([a-zA-Z0-9_.:+-]+)""", re.IGNORECASE)
17BOUNDARY_PATTERN_CACHE_SIZE: typing.Final = 32
18_CHARSET_LOOKUP_LIMIT: typing.Final = 4096
19_BOM_CHARACTER: typing.Final = "\ufeff"
20_BYTE_ORDER_MARKS: typing.Final[tuple[tuple[bytes, str], ...]] = (
21 (codecs.BOM_UTF32_LE, "utf-32"),
22 (codecs.BOM_UTF32_BE, "utf-32"),
23 (codecs.BOM_UTF8, "utf-8-sig"),
24 (codecs.BOM_UTF16_LE, "utf-16"),
25 (codecs.BOM_UTF16_BE, "utf-16"),
26)
27_META_DEPENDENT_PARTS: typing.Final[frozenset[structs.WhatToParse]] = frozenset(
28 (
29 structs.WhatToParse.OPEN_GRAPH,
30 structs.WhatToParse.TWITTER,
31 structs.WhatToParse.BASIC,
32 structs.WhatToParse.OTHER,
33 )
34)
37def set_settings_for_meta_tags(new_options: structs.SettingsFromUser) -> None:
38 """Override default package options."""
39 _GLOBAL_OPTIONS_HOLDER.set(new_options)
42def resolve_active_options(options: structs.SettingsFromUser | None) -> structs.SettingsFromUser:
43 """Return explicit options or the ones installed by set_settings_for_meta_tags."""
44 return options if options is not None else _GLOBAL_OPTIONS_HOLDER.get()
47def _read_using_encoding(raw_source: bytes, encoding_name: str) -> str | None:
48 # pages declare all kinds of nonsense: unknown names and non text codecs (charset="base64") raise
49 # LookupError, and a few stdlib codecs (charset="idna") raise UnicodeError on arbitrary bytes
50 try:
51 return raw_source.decode(encoding_name, errors="ignore")
52 except (LookupError, ValueError):
53 return None
56def convert_source_to_text(raw_source: bytes) -> str:
57 """Decode raw page bytes honouring a byte order mark or a declared charset."""
58 for one_byte_order_mark, one_bom_encoding in _BYTE_ORDER_MARKS:
59 if raw_source.startswith(one_byte_order_mark):
60 return raw_source.decode(one_bom_encoding, errors="ignore")
61 found_charset: typing.Final[re.Match[bytes] | None] = _CHARSET_DECLARATION_PATTERN.search(
62 raw_source[:_CHARSET_LOOKUP_LIMIT]
63 )
64 if found_charset is None:
65 return raw_source.decode(errors="ignore")
66 decoded_source: typing.Final[str | None] = _read_using_encoding(
67 raw_source, found_charset.group(1).decode("ascii", errors="ignore")
68 )
69 if decoded_source is None:
70 return raw_source.decode(errors="ignore")
71 return decoded_source
74@functools.lru_cache(maxsize=BOUNDARY_PATTERN_CACHE_SIZE)
75def _build_boundary_pattern(boundary_tags: tuple[str, ...]) -> re.Pattern[str]:
76 # only escaped literals end up in the alternation, so there is nothing here to backtrack on
77 return re.compile("|".join(re.escape(one_boundary_tag) for one_boundary_tag in boundary_tags), re.IGNORECASE)
80def _find_boundary_cut_position(found_boundary: re.Match[str], head_closing_tag: str) -> int:
81 # the head closing tag is kept in the window, an opening body tag is not
82 if found_boundary.group().lower() == head_closing_tag.lower():
83 return found_boundary.end()
84 return found_boundary.start()
87def _extract_html_scan_window(html_source: str, active_options: structs.SettingsFromUser) -> str:
88 # searching the original text (instead of a lowercased copy) keeps offsets valid: str.lower() is
89 # not length preserving, U+0130 for example lowercases into two characters and shifts everything
90 found_boundary: typing.Final[re.Match[str] | None] = (
91 _build_boundary_pattern(tuple(active_options.boundary_tags)).search(
92 html_source, 0, active_options.max_scan_chars
93 )
94 if active_options.boundary_tags
95 else None
96 )
97 cut_position: typing.Final[int] = (
98 _find_boundary_cut_position(found_boundary, active_options.boundary_tags[0])
99 if found_boundary is not None
100 else active_options.fallback_limit_chars
101 )
102 if active_options.hard_limit_chars is None:
103 return html_source[:cut_position]
104 return html_source[: min(cut_position, active_options.hard_limit_chars)]
107def _find_social_tag_name(
108 one_attr_group: dict[str, structs.ValuesGroup],
109 parsing_settings: typing.Mapping[str, str | tuple[str, ...]],
110) -> str:
111 tech_keys: typing.Final[KeysView[str]] = one_attr_group.keys()
112 tag_prefix: typing.Final[str] = str(parsing_settings["prefix"])
113 matching_names: typing.Final[list[str]] = [
114 one_attr_group[one_prop_name].normalized.removeprefix(tag_prefix)
115 for one_prop_name in parsing_settings["prop"]
116 if one_prop_name in tech_keys and one_attr_group[one_prop_name].normalized.startswith(tag_prefix)
117 ]
118 return matching_names[0] if matching_names else ""
121def _extract_social_tags_from_precursor(
122 all_tech_attrs: list[dict[str, structs.ValuesGroup]],
123 media_type: typing.Literal[structs.WhatToParse.OPEN_GRAPH, structs.WhatToParse.TWITTER],
124) -> list[structs.OneMetaTag]:
125 parsing_settings: typing.Final[typing.Mapping[str, str | tuple[str, ...]]] = structs.SETTINGS_FOR_SOCIAL_MEDIA[
126 media_type
127 ]
128 output_buffer: typing.Final[list[structs.OneMetaTag]] = []
129 for one_attr_group in all_tech_attrs:
130 found_tag_name = _find_social_tag_name(one_attr_group, parsing_settings)
131 if found_tag_name and "content" in one_attr_group and one_attr_group["content"].original:
132 output_buffer.append(structs.OneMetaTag(name=found_tag_name, value=one_attr_group["content"].original))
133 return output_buffer
136def _extract_basic_tags_from_precursor(
137 all_tech_attrs: list[dict[str, structs.ValuesGroup]],
138) -> list[structs.OneMetaTag]:
139 collected_basic_tags: typing.Final[dict[str, str]] = {}
140 for one_attr_group in all_tech_attrs:
141 tech_keys: KeysView[str] = one_attr_group.keys()
142 if len(collected_basic_tags) == len(structs.BASIC_META_TAGS):
143 break
144 if "name" not in tech_keys or "content" not in tech_keys:
145 continue
146 basic_tag_name: str = one_attr_group["name"].normalized
147 # duplicated basic tags are common in the wild, the first one wins and the rest are dropped
148 if basic_tag_name not in structs.BASIC_META_TAGS or basic_tag_name in collected_basic_tags:
149 continue
150 if one_attr_group["content"].original:
151 collected_basic_tags[basic_tag_name] = one_attr_group["content"].original
152 return [
153 structs.OneMetaTag(name=one_tag_name, value=one_tag_value)
154 for one_tag_name, one_tag_value in collected_basic_tags.items()
155 ]
158def _match_social_prefix(one_attr_group: dict[str, structs.ValuesGroup], tech_keys: KeysView[str]) -> bool:
159 return any(
160 one_prop_name in tech_keys and one_attr_group[one_prop_name].normalized.startswith(one_config["prefix"])
161 for one_config in structs.SETTINGS_FOR_SOCIAL_MEDIA.values()
162 for one_prop_name in one_config["prop"]
163 )
166def _extract_all_other_tags_from_precursor(
167 all_tech_attrs: list[dict[str, structs.ValuesGroup]],
168) -> list[structs.OneMetaTag]:
169 output_buffer: typing.Final[list[structs.OneMetaTag]] = []
170 for one_attr_group in all_tech_attrs:
171 tech_keys: KeysView[str] = one_attr_group.keys()
172 if _match_social_prefix(one_attr_group, tech_keys):
173 continue
174 if "name" not in tech_keys or one_attr_group["name"].normalized in structs.BASIC_META_TAGS:
175 continue
176 if "content" in one_attr_group and one_attr_group["content"].original:
177 output_buffer.append(
178 structs.OneMetaTag(
179 name=one_attr_group["name"].normalized,
180 value=one_attr_group["content"].original,
181 )
182 )
183 return output_buffer
186def _prepare_normalized_meta_attrs(html_tree: LexborHTMLParser) -> list[dict[str, structs.ValuesGroup]]:
187 normalized_meta_attrs: typing.Final[list[dict[str, structs.ValuesGroup]]] = []
188 for one_meta_node in html_tree.css("meta"):
189 prepared_attrs: dict[str, structs.ValuesGroup] = {}
190 for attr_name, raw_value in one_meta_node.attributes.items():
191 prepared_value: str = raw_value or ""
192 prepared_attrs[attr_name.lower().strip()] = structs.ValuesGroup(
193 original=prepared_value,
194 normalized=prepared_value.lower().strip(),
195 )
196 normalized_meta_attrs.append(prepared_attrs)
197 return normalized_meta_attrs
200def parse_meta_tags_from_source(
201 source_code: str | bytes,
202 *,
203 options: structs.SettingsFromUser | None = None,
204) -> structs.TagsGroup:
205 normalized_source: typing.Final[str] = (
206 convert_source_to_text(source_code) if isinstance(source_code, bytes) else source_code.lstrip(_BOM_CHARACTER)
207 )
208 active_options: typing.Final[structs.SettingsFromUser] = resolve_active_options(options)
209 requested_parts: typing.Final[frozenset[structs.WhatToParse]] = frozenset(active_options.what_to_parse)
210 html_tree: typing.Final[LexborHTMLParser] = LexborHTMLParser(
211 _extract_html_scan_window(normalized_source, active_options)
212 if active_options.optimize_input
213 else normalized_source
214 )
215 title_node: typing.Final[LexborNode | None] = (
216 html_tree.css_first("title") if structs.WhatToParse.TITLE in requested_parts else None
217 )
218 page_title: typing.Final[str] = title_node.text().strip() if title_node else ""
219 normalized_meta_attrs: typing.Final[list[dict[str, structs.ValuesGroup]]] = (
220 _prepare_normalized_meta_attrs(html_tree) if requested_parts & _META_DEPENDENT_PARTS else []
221 )
223 open_graph_meta_tags: typing.Final[list[structs.OneMetaTag]] = (
224 _extract_social_tags_from_precursor(normalized_meta_attrs, structs.WhatToParse.OPEN_GRAPH)
225 if structs.WhatToParse.OPEN_GRAPH in requested_parts
226 else []
227 )
228 twitter_meta_tags: typing.Final[list[structs.OneMetaTag]] = (
229 _extract_social_tags_from_precursor(normalized_meta_attrs, structs.WhatToParse.TWITTER)
230 if structs.WhatToParse.TWITTER in requested_parts
231 else []
232 )
233 basic_meta_tags: typing.Final[list[structs.OneMetaTag]] = (
234 _extract_basic_tags_from_precursor(normalized_meta_attrs)
235 if structs.WhatToParse.BASIC in requested_parts
236 else []
237 )
238 other_meta_tags: typing.Final[list[structs.OneMetaTag]] = (
239 _extract_all_other_tags_from_precursor(normalized_meta_attrs)
240 if structs.WhatToParse.OTHER in requested_parts
241 else []
242 )
244 return structs.TagsGroup(
245 title=page_title,
246 basic=basic_meta_tags,
247 open_graph=open_graph_meta_tags,
248 twitter=twitter_meta_tags,
249 other=other_meta_tags,
250 )