diff --git a/readability/readability.py b/readability/readability.py index 15c86c1..00f25fd 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -191,10 +191,27 @@ def _parse(self, input): self.encoding = 'utf-8' else: doc, self.encoding = build_doc(input) + + def hide_root(root): + for child in list(root): + if child.tag != "head": + child.drop_tree() + # Fragment roots can contain hidden text outside child elements. + if root.tag != "html" and root.text and not root.text.isspace(): + root.text = None + root.attrib.pop("hidden", None) + root.attrib.pop("style", None) + for elem in doc.xpath("//*[@hidden]"): + if elem is doc or elem.getparent() is None: + hide_root(elem) + continue elem.drop_tree() for elem in doc.xpath("//*[@style]"): if REGEXES["displayNoneRe"].search(elem.get("style")): + if elem is doc or elem.getparent() is None: + hide_root(elem) + continue elem.drop_tree() doc = html_cleaner.clean_html(doc) base_href = self.url diff --git a/tests/test_article_only.py b/tests/test_article_only.py index cb8d1ff..8a874e5 100644 --- a/tests/test_article_only.py +++ b/tests/test_article_only.py @@ -2,6 +2,9 @@ import time import unittest +import pytest +from lxml.html import document_fromstring, fromstring + from readability import Document from functools import wraps @@ -40,6 +43,103 @@ def load_sample(filename): return html +@pytest.mark.parametrize("attribute", [ + "hidden", + 'style="display:none"', + 'style="visibility:hidden"', + 'hidden style="display:none"', +]) +@pytest.mark.parametrize("method", ["content", "summary", "title"]) +@pytest.mark.parametrize("make_input", [str, str.encode, document_fromstring], + ids=["text", "bytes", "element"]) +@pytest.mark.parametrize("hidden_descendants", [False, True]) +@pytest.mark.parametrize("head, expected_title", [ + ("", "[no-title]"), + ("T", "T"), + ('T' + 'Hidden styled title', "T"), +], ids=["without-head", "with-title", "with-hidden-head-nodes"]) +@pytest.mark.parametrize("formatted", [False, True]) +def test_hidden_root_matches_hidden_body( + attribute, method, make_input, hidden_descendants, head, expected_title, + formatted +): + newline = "\n" if formatted else "" + descendants = ' hidden style="display:none"' if hidden_descendants else "" + root_hidden = ( + f"{newline}{head}{newline}" + f"{newline}Hidden content

{newline}" + f"{newline}" + ) + body_hidden = ( + f"{newline}{head}{newline}" + f"{newline}

Hidden content

{newline}{newline}" + ) + + try: + expected = getattr(Document(make_input(body_hidden)), method)() + except IndexError as error: + # lxml 5.4 raises IndexError when accessing .body on an empty document. + # Preserve the existing content() behavior for a hidden body. + assert method == "content" + assert str(error) == "list index out of range" + with pytest.raises(IndexError, match="list index out of range"): + getattr(Document(make_input(root_hidden)), method)() + else: + result = getattr(Document(make_input(root_hidden)), method)() + assert result == expected + assert "Hidden" not in result + if method == "title": + assert result == expected_title + + +@pytest.mark.parametrize("attribute", [ + "hidden", + 'style="display:none"', + 'style="visibility:hidden"', + 'hidden style="display:none"', +]) +@pytest.mark.parametrize("method", ["content", "summary", "title"]) +@pytest.mark.parametrize("direct_text", ["", "Hidden text"]) +def test_hidden_fragment_root(attribute, method, direct_text): + sample = fromstring( + f'
{direct_text}' + "Hidden tail
" + ) + control = fromstring("
") + + try: + expected = getattr(Document(control), method)() + except IndexError as error: + assert method == "content" + assert str(error) == "list index out of range" + with pytest.raises(IndexError, match="list index out of range"): + getattr(Document(sample), method)() + else: + result = getattr(Document(sample), method)() + assert result == expected + assert "Hidden" not in result + + +@pytest.mark.parametrize("attribute", [ + "hidden", + 'style="display:none"', + 'style="visibility:hidden"', + 'hidden style="display:none"', +]) +@pytest.mark.parametrize("method", ["content", "summary"]) +def test_hidden_child_preserves_visible_text_and_tail(attribute, method): + sample = ( + f"

Visible beforeHidden content" + " visible after.

" + ) + + result = getattr(Document(sample), method)() + + assert "Hidden content" not in result + assert "Visible before visible after." in result + + class TestArticleOnly(unittest.TestCase): """The option to not get back a full html doc should work