Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions ricecooker/utils/downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,16 @@ def make_request(

_CSS_URL_RE = re.compile(r"url\(['\"]?(.*?)['\"]?\)")

# Handle CSS imports with a string rather than a url()
# From MDN docs on @import:
# https://developer.mozilla.org/en-US/docs/Web/CSS/Reference/At-rules/@import
# url - Is a <string> or a <url> type representing the location of the resource to import.
# The URL may be absolute or relative.
# Note that this regex is for the specialized import case that can be given a regular
# string with a URL in it. An @import given a value of type <url>, will be handled
# by the above regex and associated code.
_CSS_IMPORT_RE = re.compile(r"@import\s*['\"](.*?)['\"]")


# TODO(davidhu): Use MD5 hash of URL (ideally file) instead.
def _derive_filename(url):
Expand Down Expand Up @@ -443,12 +453,18 @@ def css_content_middleware(content, url, **kwargs):

root_parts = urlparse(url)

def repl_url(match):
return "url(%s)" % fix_url(match)

def repl_import(match):
return "@import%s" % fix_url(match)

# Download linked fonts and images
def repl(match):
def fix_url(match):
src = match.group(1)

if src.startswith("//localhost"):
return "url()"
return ""
# Don't download data: files
if src.startswith("data:"):
return match.group(0)
Expand All @@ -470,7 +486,7 @@ def repl(match):

if _is_blacklisted(src_url, url_blacklist):
print(" Skipping downloading blacklisted url", src_url)
return "url()"
return ""

derived_filename = derive_filename(src_url)

Expand All @@ -497,9 +513,10 @@ def repl(match):
LOGGER.debug(
"Resource already downloaded, skipping: {}".format(src_url)
)
return 'url("%s")' % new_url
return '"%s"' % new_url

return _CSS_URL_RE.sub(repl, content)
urls_rewritten = _CSS_URL_RE.sub(repl_url, content)
return _CSS_IMPORT_RE.sub(repl_import, urls_rewritten)

if link_policy is not None and "blacklist" in link_policy:
url_blacklist += link_policy["blacklist"]
Expand Down
64 changes: 64 additions & 0 deletions tests/test_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,3 +171,67 @@ def test_pretextbook_css_fetch():
/ "material_symbols.woff"
)
assert font_path.stat().st_size > 0


class _FakeResponse:
"""Minimal requests.Response stand-in for an injected request_fn."""

def __init__(self, body, content_type="text/css"):
self.content = body.encode("utf-8") if isinstance(body, str) else body
self.headers = {"content-type": content_type}
self.status_code = 200
self.encoding = "utf-8"
self.url = None

@property
def text(self):
return self.content.decode(self.encoding or "utf-8")

def raise_for_status(self):
pass


def test_css_import_bare_string_is_archived():
"""A CSS @import written as a bare string is downloaded and rewritten.

``@import "theme.css"`` references a stylesheet the same way
``@import url("theme.css")`` does, but the rewriter only recognised the
``url()`` form, so a bare-string import was neither downloaded nor relinked.
"""
page_url = "https://example.org/index.html"
bodies = {
"https://example.org/main.css": '@import "theme.css";\nbody { color: red; }',
"https://example.org/theme.css": "h1 { color: blue; }",
}

def fake_request(url):
return _FakeResponse(bodies[url])

html = '<html><head><link rel="stylesheet" href="main.css"/></head></html>'

with tempfile.TemporaryDirectory() as download_root:
resource_urls = {}

def derive_filename(url):
return downloader.get_archive_filename(
url, page_url, download_root, resource_urls
)

downloader.download_static_assets(
html,
download_root,
"https://example.org/",
request_fn=fake_request,
derive_filename=derive_filename,
resource_urls=resource_urls,
relative_links=True,
)

root = Path(download_root)
# The imported stylesheet is downloaded to its archive path...
assert (root / "example.org" / "theme.css").is_file()
# ...and the @import in main.css now points at the local copy, not the
# original remote URL.
main_css = (root / "example.org" / "main.css").read_text()
assert "theme.css" in main_css
assert "https://example.org/theme.css" not in main_css
Loading