diff --git a/readability/readability.py b/readability/readability.py index 15c86c1..00f25fd 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -191,10 +191,27 @@ def _parse(self, input): self.encoding = 'utf-8' else: doc, self.encoding = build_doc(input) + + def hide_root(root): + for child in list(root): + if child.tag != "head": + child.drop_tree() + # Fragment roots can contain hidden text outside child elements. + if root.tag != "html" and root.text and not root.text.isspace(): + root.text = None + root.attrib.pop("hidden", None) + root.attrib.pop("style", None) + for elem in doc.xpath("//*[@hidden]"): + if elem is doc or elem.getparent() is None: + hide_root(elem) + continue elem.drop_tree() for elem in doc.xpath("//*[@style]"): if REGEXES["displayNoneRe"].search(elem.get("style")): + if elem is doc or elem.getparent() is None: + hide_root(elem) + continue elem.drop_tree() doc = html_cleaner.clean_html(doc) base_href = self.url diff --git a/tests/test_article_only.py b/tests/test_article_only.py index cb8d1ff..8a874e5 100644 --- a/tests/test_article_only.py +++ b/tests/test_article_only.py @@ -2,6 +2,9 @@ import time import unittest +import pytest +from lxml.html import document_fromstring, fromstring + from readability import Document from functools import wraps @@ -40,6 +43,103 @@ def load_sample(filename): return html +@pytest.mark.parametrize("attribute", [ + "hidden", + 'style="display:none"', + 'style="visibility:hidden"', + 'hidden style="display:none"', +]) +@pytest.mark.parametrize("method", ["content", "summary", "title"]) +@pytest.mark.parametrize("make_input", [str, str.encode, document_fromstring], + ids=["text", "bytes", "element"]) +@pytest.mark.parametrize("hidden_descendants", [False, True]) +@pytest.mark.parametrize("head, expected_title", [ + ("", "[no-title]"), + ("
Hidden content
{newline}" + f"{newline}" + ) + body_hidden = ( + f"{newline}{head}{newline}" + f"{newline}Hidden content
{newline}{newline}" + ) + + try: + expected = getattr(Document(make_input(body_hidden)), method)() + except IndexError as error: + # lxml 5.4 raises IndexError when accessing .body on an empty document. + # Preserve the existing content() behavior for a hidden body. + assert method == "content" + assert str(error) == "list index out of range" + with pytest.raises(IndexError, match="list index out of range"): + getattr(Document(make_input(root_hidden)), method)() + else: + result = getattr(Document(make_input(root_hidden)), method)() + assert result == expected + assert "Hidden" not in result + if method == "title": + assert result == expected_title + + +@pytest.mark.parametrize("attribute", [ + "hidden", + 'style="display:none"', + 'style="visibility:hidden"', + 'hidden style="display:none"', +]) +@pytest.mark.parametrize("method", ["content", "summary", "title"]) +@pytest.mark.parametrize("direct_text", ["", "Hidden text"]) +def test_hidden_fragment_root(attribute, method, direct_text): + sample = fromstring( + f'Hidden child
' + "Hidden tailVisible beforeHidden content" + " visible after.
" + ) + + result = getattr(Document(sample), method)() + + assert "Hidden content" not in result + assert "Visible before visible after." in result + + class TestArticleOnly(unittest.TestCase): """The option to not get back a full html doc should work