Coverage for tests/reference_parser.py: 100%

1"""A deliberately naive reference implementation used as an independent oracle. 

2 

3It is built on the standard library ``html.parser`` instead of selectolax, and it re-implements the 

4documented rules of this package from scratch: what counts as an Open Graph tag, a Twitter tag, a 

5basic tag and an "other" tag, and where the head ends. Real pages are then parsed twice, once by the 

6package and once by this module, and the two results must agree. A snapshot cannot catch a 

7regression that also updates the snapshot, two independent implementations can. 

8""" 

9 

10import dataclasses 

11import html.parser 

12import typing 

13 

14 

15BASIC_META_TAG_NAMES: typing.Final[tuple[str, ...]] = ("title", "description", "keywords", "robots", "viewport") 

16HEAD_ENDING_TAGS: typing.Final[frozenset[str]] = frozenset(("head", "body")) 

17 

18 

19@typing.final 

20@dataclasses.dataclass(kw_only=True, slots=True, frozen=True) 

21class ReferenceResult: 

22 """What the reference implementation found inside the head of a page.""" 

23 

24 page_title: str 

25 open_graph_tags: list[list[str]] 

26 twitter_tags: list[list[str]] 

27 basic_tags: list[list[str]] 

28 other_tags: list[list[str]] 

29 

30 

31@typing.final 

32class ReferenceHeadParser(html.parser.HTMLParser): 

33 """Collect the title and every meta tag until the head is over.""" 

34 

35 def __init__(self) -> None: 

36 super().__init__(convert_charrefs=True) 

37 self.collected_meta: list[dict[str, str]] = [] 

38 self.collected_title: str | None = None 

39 self.head_is_over: bool = False 

40 self.inside_title: bool = False 

41 

42 def handle_starttag(self, tag_name: str, tag_attributes: list[tuple[str, str | None]]) -> None: 

43 if self.head_is_over: 

44 return 

45 if tag_name == "body": 

46 self.head_is_over = True 

47 return 

48 if tag_name == "title" and self.collected_title is None: 

49 self.inside_title = True 

50 self.collected_title = "" 

51 return 

52 if tag_name == "meta": 

53 self.collected_meta.append( 

54 {one_name.lower().strip(): (one_value or "") for one_name, one_value in tag_attributes} 

55 ) 

56 

57 def handle_endtag(self, tag_name: str) -> None: 

58 if tag_name in HEAD_ENDING_TAGS: 

59 self.head_is_over = True 

60 if tag_name == "title": 

61 self.inside_title = False 

62 

63 def handle_data(self, data_text: str) -> None: 

64 if self.inside_title and self.collected_title is not None: 

65 self.collected_title += data_text 

66 

67 

68def _read_attribute(one_meta: dict[str, str], attribute_name: str) -> str: 

69 return one_meta.get(attribute_name, "").lower().strip() 

70 

71 

72def _collect_social_tags( 

73 collected_meta: list[dict[str, str]], tag_prefix: str, *, allowed_attributes: tuple[str, ...] 

74) -> list[list[str]]: 

75 collected_pairs: typing.Final[list[list[str]]] = [] 

76 for one_meta in collected_meta: 

77 matching_names = [ 

78 _read_attribute(one_meta, one_attribute).removeprefix(tag_prefix) 

79 for one_attribute in allowed_attributes 

80 if _read_attribute(one_meta, one_attribute).startswith(tag_prefix) 

81 ] 

82 if matching_names and one_meta.get("content"): 

83 collected_pairs.append([matching_names[0], one_meta["content"]]) 

84 return collected_pairs 

85 

86 

87def _collect_basic_tags(collected_meta: list[dict[str, str]]) -> list[list[str]]: 

88 collected_values: typing.Final[dict[str, str]] = {} 

89 for one_meta in collected_meta: 

90 tag_name = _read_attribute(one_meta, "name") 

91 if "name" not in one_meta or tag_name not in BASIC_META_TAG_NAMES or not one_meta.get("content"): 

92 continue 

93 collected_values.setdefault(tag_name, one_meta["content"]) 

94 return [[one_name, one_value] for one_name, one_value in collected_values.items()] 

95 

96 

97def _collect_other_tags(collected_meta: list[dict[str, str]]) -> list[list[str]]: 

98 return [ 

99 [_read_attribute(one_meta, "name"), one_meta["content"]] 

100 for one_meta in collected_meta 

101 if "name" in one_meta 

102 and one_meta.get("content") 

103 and not _read_attribute(one_meta, "name").startswith("twitter:") 

104 and not _read_attribute(one_meta, "property").startswith(("twitter:", "og:")) 

105 and _read_attribute(one_meta, "name") not in BASIC_META_TAG_NAMES 

106 ] 

107 

108 

109def extract_reference_tags(page_text: str) -> ReferenceResult: 

110 """Parse the head of a page with the standard library and apply this package's documented rules.""" 

111 reference_parser: typing.Final = ReferenceHeadParser() 

112 # the html specification normalizes newlines while preprocessing the input stream 

113 reference_parser.feed(page_text.replace("\r\n", "\n").replace("\r", "\n")) 

114 reference_parser.close() 

115 collected_meta: typing.Final[list[dict[str, str]]] = reference_parser.collected_meta 

116 return ReferenceResult( 

117 page_title=(reference_parser.collected_title or "").strip(), 

118 open_graph_tags=_collect_social_tags(collected_meta, "og:", allowed_attributes=("property",)), 

119 twitter_tags=_collect_social_tags(collected_meta, "twitter:", allowed_attributes=("name", "property")), 

120 basic_tags=_collect_basic_tags(collected_meta), 

121 other_tags=_collect_other_tags(collected_meta), 

122 )