From fdeb79f5e494a6235d992acf7a7a25f3f8e928c9 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 3 Mar 2021 23:58:13 +0100 Subject: [PATCH 1/7] Add feature to bypass Content Security Policy As pointed out by @deatondg in #6, some sites have a CSP that prevents userscripts from running properly. This PR makes it possible to either bypass the CSP specifically for any userscripts that are injected (inline or not) or disable the CSP altogether whenever a userscript is injected. The latter is often necessary because userscripts tend to inject at least one resource into the page, be it an external image or just some inline CSS. Resolves #6. Co-authored-by: deatondg --- src/injector.py | 33 +++++++++++++++++++++++++++++---- src/launcher.py | 2 ++ src/modules/argparser.py | 8 ++++++++ src/modules/arguments.py | 9 +++++++++ src/modules/csp.py | 40 ++++++++++++++++++++++++++++++++++++++++ src/modules/inject.py | 2 ++ 6 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 src/modules/csp.py diff --git a/src/injector.py b/src/injector.py index f3fc8cb..c5b7da2 100644 --- a/src/injector.py +++ b/src/injector.py @@ -9,6 +9,7 @@ import modules.arguments as A import modules.constants as C +import modules.csp as csp import modules.inject as inject import modules.inline as inline import modules.metadata as metadata @@ -139,6 +140,7 @@ def load(self, loader): loader.add_option(sanitize(A.inline), bool, False, A.inline_help) loader.add_option(sanitize(A.no_default_userscripts), bool, False, A.no_default_userscripts_help) loader.add_option(sanitize(A.list_injected), bool, False, A.list_injected_help) + loader.add_option(sanitize(A.bypass_csp), Optional[str], None, A.bypass_csp_help) loader.add_option(sanitize(A.userscripts_dir), Optional[str], A.userscripts_dir_default, A.userscripts_dir_help) loader.add_option(sanitize(A.query_param_to_disable), str, A.query_param_to_disable_default, A.query_param_to_disable_help) @@ -167,7 +169,7 @@ def response(self, flow: http.HTTPFlow): if CONTENT_TYPE in response.headers: if any(map(lambda t: t in response.headers[CONTENT_TYPE], RELEVANT_CONTENT_TYPES)): # Response is a web page; proceed. - insertedScripts: List[str] = [] + injections: List[csp.Injection] = [] soup = BeautifulSoup( response.content, HTML_PARSER, @@ -185,23 +187,31 @@ def response(self, flow: http.HTTPFlow): logError(unsafeSequencesMessage(script)) continue logInfo(f"""Injecting {script.name}{"" if script.version is None else " " + C.VERSION_PREFIX + script.version} into {requestURL} ({"inline" if useInline else "linked"}) ...""") + nonce = csp.generateNonce() result = inject.inject(script, soup, inject.Options( inline = option(A.inline), + nonce = nonce )) if type(result) is BeautifulSoup: soup = result - insertedScripts.append(script.name + ("" if script.version is None else " " + T.stringifyVersion(script.version))) + injections.append(csp.Injection( + userscript = script, + useInline = useInline, + nonce = nonce, + )) else: logError("Injection failed due to the following error:") logError(str(result)) + handleContentSecurityPolicy(response, injections) index_DTD: Optional[int] = indexOfDTD(soup) # Insert information comment: if option(A.list_injected): + namesOfInjectedScripts = [ i.userscript.name + ("" if i.userscript.version is None else " " + T.stringifyVersion(i.userscript.version)) for i in injections ] soup.insert(0 if index_DTD is None else 1+index_DTD, Comment( HTML_INFO_COMMENT_PREFIX + ( - "No matching userscripts for this URL." if insertedScripts == [] - else "These scripts were inserted:\n" + bulletList(insertedScripts) + "No matching userscripts for this URL." if namesOfInjectedScripts == [] + else "These scripts were inserted:\n" + bulletList(namesOfInjectedScripts) ) + "\n" )) # Serialize and encode: @@ -211,4 +221,19 @@ def response(self, flow: http.HTTPFlow): ) +def handleContentSecurityPolicy(response: http.HTTPFlow.response, injections: List[csp.Injection]): + # If there is a CSP header, we may need to modify it for the userscript(s) to work. + ContentSecurityPolicy = "Content-Security-Policy" + if ContentSecurityPolicy in response.headers: + bypassCspValue = option(A.bypass_csp) + if bypassCspValue == A.bypass_csp_script: + logInfo(f"Bypassing host site's Content Security Policy for userscripts only (not any resources injected _by_ userscripts, such as stylesheets and images). Try `{flag(A.bypass_csp)} {A.bypass_csp_everything}` if something does not work properly.") + response.headers[ContentSecurityPolicy] = csp.headerWithScriptsAllowed(response.headers[ContentSecurityPolicy], injections) + elif bypassCspValue == A.bypass_csp_everything: + logInfo(f"Bypassing host site's Content Security Policy altogether due to `{flag(A.bypass_csp)} {A.bypass_csp_everything}`.") + del response.headers[ContentSecurityPolicy] + else: + logWarning(f"Host site has a Content Security Policy. Try the {flag(A.bypass_csp)} flag if userscripts don't work properly.") + + addons = [ UserscriptInjector() ] diff --git a/src/launcher.py b/src/launcher.py index f9804a8..f868cb8 100755 --- a/src/launcher.py +++ b/src/launcher.py @@ -55,6 +55,7 @@ def checkThatUserscriptsDirectoryExistsIfSpecified(directory: str): useTransparent = args.transparent useFiltering = useCustomFiltering or useDefaultRules useIntercept = args.intercept is True + bypassCsp = args.bypass_csp userscriptsDirectory = args.userscripts_dir checkThatUserscriptsDirectoryExistsIfSpecified(userscriptsDirectory) def ruleFilesContent_default(): @@ -101,6 +102,7 @@ def ruleFilesContent_custom(): "--set", f"""{sanitize(A.inline)}={str(args.inline).lower()}""", "--set", f"""{sanitize(A.list_injected)}={str(args.list_injected).lower()}""", "--set", f"""{sanitize(A.no_default_userscripts)}={str(args.no_default_userscripts).lower()}""", + "--set", "" if bypassCsp is None else f"""{sanitize(A.bypass_csp)}={bypassCsp}""", "--set", "" if userscriptsDirectory is None else f"""{sanitize(A.userscripts_dir)}={userscriptsDirectory}""", "--set", f"""{sanitize(A.query_param_to_disable)}={args.query_param_to_disable}""", # Empty string breaks the argument chain: diff --git a/src/modules/argparser.py b/src/modules/argparser.py index 2c34f83..4a7f525 100644 --- a/src/modules/argparser.py +++ b/src/modules/argparser.py @@ -6,6 +6,14 @@ def getArgparser(): argparser = ArgumentParser(description=T.description) + argparser.add_argument( + flag(A.bypass_csp), + type=str, + metavar=A.metavar_allow, + choices=A.bypass_csp_values, + default=A.bypass_csp_default, + help=A.bypass_csp_help, + ) argparser.add_argument( flag(A.intercept), action="store_true", diff --git a/src/modules/arguments.py b/src/modules/arguments.py index 7a3fe6e..3f8bd4e 100644 --- a/src/modules/arguments.py +++ b/src/modules/arguments.py @@ -4,9 +4,18 @@ metavar_file = "FILE" metavar_dir = "DIR" metavar_param = "PARAM" +metavar_allow = "ALLOW" RULES = "rules" +bypass_csp = "bypass-csp" +bypass_csp_never = "never" +bypass_csp_script = "script" +bypass_csp_everything = "everything" +bypass_csp_default = bypass_csp_never +bypass_csp_values = { bypass_csp_never, bypass_csp_script, bypass_csp_everything } +bypass_csp_help = f"Bypass host site's Content Security Policy to allow userscripts to run properly. If {metavar_allow} is '{bypass_csp_script}', the CSP is bypassed only for the userscript itself. Use '{bypass_csp_everything}' to allow everything, which may be necessary if the userscript injects CSS, images etc. Note that the latter completely disables any CSP from every host site into which a userscript is injected. Default: '{bypass_csp_default}'." + inline = "inline" inline_short = "i" inline_help = "Always insert userscripts inline, never linked" diff --git a/src/modules/csp.py b/src/modules/csp.py new file mode 100644 index 0000000..c7a187e --- /dev/null +++ b/src/modules/csp.py @@ -0,0 +1,40 @@ +import secrets +from typing import List, NamedTuple + +from modules.userscript import Userscript + +# Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy + + +class Injection(NamedTuple): + userscript: Userscript + useInline: bool + nonce: str + + +def headerWithScriptsAllowed(cspHeaderValue: str, injections: List[Injection]) -> str: + # Example CSP header: + # + # Content-Security-Policy: default-src 'self'; frame-src 'self'; img-src https:; connect-src 'self' + # + cspKeyValuePairs = [ directive.strip().split(" ", 1) for directive in cspHeaderValue.split(';') ] + cspDict = { key: value for key, value in cspKeyValuePairs } + if "script-src" not in cspDict: + # Browsers fall back to default-src if there is no script-src. + # Since there was no script-src directive and we are adding one, we include the default-src (if present) in it to avoid breaking the site's effective CSP. + cspDict["script-src"] = cspDict["default-src"] if "default-src" in cspDict else "" + sourcesToAllow = [ source(i) for i in injections ] + cspDict["script-src"] += " " + " ".join(sourcesToAllow) + return '; '.join([ f'{key} {value}' for key, value in cspDict.items() ]) + + +def source(injection: Injection) -> str: + if injection.useInline: + return f"'nonce-{injection.nonce}'" + else: + # MDN about host (i.e. download URL) sources: "Unlike other values below, single quotes shouldn't be used." + return injection.userscript.downloadURL + + +def generateNonce(): + return secrets.token_hex() # If no argument is passed, "a reasonable default is used" for the number of bytes. diff --git a/src/modules/inject.py b/src/modules/inject.py index b6de896..957b749 100644 --- a/src/modules/inject.py +++ b/src/modules/inject.py @@ -9,11 +9,13 @@ class Options(NamedTuple): inline: bool + nonce: str def inject(script: Userscript, soup: BeautifulSoup, options: Options) -> Union[BeautifulSoup, Exception]: useInline = options.inline or script.downloadURL is None tag = soup.new_tag("script") + tag["nonce"] = options.nonce # Used to bypass CSP for inline-injected userscripts. tag[C.ATTRIBUTE_UP_VERSION] = C.VERSION withLoadListenerIfRunAtIdle = userscript.withEventListener("load") if script.runAt == document_idle else idem withNoframesIfNoframes = userscript.withNoframes if script.noframes else idem From 28e49d3f0f3b307897037f62a1bc0ae9852480b8 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 21 Mar 2021 19:44:03 +0100 Subject: [PATCH 2/7] Fix default value in loader.add_option call --- src/injector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/injector.py b/src/injector.py index c5b7da2..336e4f8 100644 --- a/src/injector.py +++ b/src/injector.py @@ -140,7 +140,7 @@ def load(self, loader): loader.add_option(sanitize(A.inline), bool, False, A.inline_help) loader.add_option(sanitize(A.no_default_userscripts), bool, False, A.no_default_userscripts_help) loader.add_option(sanitize(A.list_injected), bool, False, A.list_injected_help) - loader.add_option(sanitize(A.bypass_csp), Optional[str], None, A.bypass_csp_help) + loader.add_option(sanitize(A.bypass_csp), Optional[str], A.bypass_csp_default, A.bypass_csp_help) loader.add_option(sanitize(A.userscripts_dir), Optional[str], A.userscripts_dir_default, A.userscripts_dir_help) loader.add_option(sanitize(A.query_param_to_disable), str, A.query_param_to_disable_default, A.query_param_to_disable_help) From 1a96f3279691776c627a3b3819eea3865d859ebb Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 21 Mar 2021 19:47:45 +0100 Subject: [PATCH 3/7] Rename 'never' to 'nothing' --- src/modules/arguments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/modules/arguments.py b/src/modules/arguments.py index 3f8bd4e..265ba68 100644 --- a/src/modules/arguments.py +++ b/src/modules/arguments.py @@ -9,11 +9,11 @@ RULES = "rules" bypass_csp = "bypass-csp" -bypass_csp_never = "never" +bypass_csp_nothing = "nothing" bypass_csp_script = "script" bypass_csp_everything = "everything" -bypass_csp_default = bypass_csp_never -bypass_csp_values = { bypass_csp_never, bypass_csp_script, bypass_csp_everything } +bypass_csp_default = bypass_csp_nothing +bypass_csp_values = { bypass_csp_nothing, bypass_csp_script, bypass_csp_everything } bypass_csp_help = f"Bypass host site's Content Security Policy to allow userscripts to run properly. If {metavar_allow} is '{bypass_csp_script}', the CSP is bypassed only for the userscript itself. Use '{bypass_csp_everything}' to allow everything, which may be necessary if the userscript injects CSS, images etc. Note that the latter completely disables any CSP from every host site into which a userscript is injected. Default: '{bypass_csp_default}'." inline = "inline" From 679575ec192d25b29c451162e719470283716dfb Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 21 Mar 2021 19:49:00 +0100 Subject: [PATCH 4/7] Make 'script' the default value for --bypass-csp --- src/modules/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/arguments.py b/src/modules/arguments.py index 265ba68..022c2ac 100644 --- a/src/modules/arguments.py +++ b/src/modules/arguments.py @@ -12,7 +12,7 @@ bypass_csp_nothing = "nothing" bypass_csp_script = "script" bypass_csp_everything = "everything" -bypass_csp_default = bypass_csp_nothing +bypass_csp_default = bypass_csp_script bypass_csp_values = { bypass_csp_nothing, bypass_csp_script, bypass_csp_everything } bypass_csp_help = f"Bypass host site's Content Security Policy to allow userscripts to run properly. If {metavar_allow} is '{bypass_csp_script}', the CSP is bypassed only for the userscript itself. Use '{bypass_csp_everything}' to allow everything, which may be necessary if the userscript injects CSS, images etc. Note that the latter completely disables any CSP from every host site into which a userscript is injected. Default: '{bypass_csp_default}'." From 47d50fc2a365ace7e413070c83a6a750795240a0 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 21 Mar 2021 20:12:01 +0100 Subject: [PATCH 5/7] Only include nonce when necessary --- src/injector.py | 4 ++-- src/modules/csp.py | 8 ++++---- src/modules/inject.py | 9 +++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/injector.py b/src/injector.py index 336e4f8..8db983c 100644 --- a/src/injector.py +++ b/src/injector.py @@ -187,7 +187,8 @@ def response(self, flow: http.HTTPFlow): logError(unsafeSequencesMessage(script)) continue logInfo(f"""Injecting {script.name}{"" if script.version is None else " " + C.VERSION_PREFIX + script.version} into {requestURL} ({"inline" if useInline else "linked"}) ...""") - nonce = csp.generateNonce() + shouldUseNonce = useInline and option(A.bypass_csp) == A.bypass_csp_script # If not inline, then URL is used for bypassing; if bypass for nothing or everything, then the nonce would have no effect anyway. + nonce = csp.generateNonce() if shouldUseNonce else None result = inject.inject(script, soup, inject.Options( inline = option(A.inline), nonce = nonce @@ -196,7 +197,6 @@ def response(self, flow: http.HTTPFlow): soup = result injections.append(csp.Injection( userscript = script, - useInline = useInline, nonce = nonce, )) else: diff --git a/src/modules/csp.py b/src/modules/csp.py index c7a187e..9a6f41f 100644 --- a/src/modules/csp.py +++ b/src/modules/csp.py @@ -1,15 +1,15 @@ import secrets -from typing import List, NamedTuple +from typing import List, NamedTuple, Optional from modules.userscript import Userscript +from modules.utilities import isSomething # Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy class Injection(NamedTuple): userscript: Userscript - useInline: bool - nonce: str + nonce: Optional[str] def headerWithScriptsAllowed(cspHeaderValue: str, injections: List[Injection]) -> str: @@ -29,7 +29,7 @@ def headerWithScriptsAllowed(cspHeaderValue: str, injections: List[Injection]) - def source(injection: Injection) -> str: - if injection.useInline: + if isSomething(injection.nonce): return f"'nonce-{injection.nonce}'" else: # MDN about host (i.e. download URL) sources: "Unlike other values below, single quotes shouldn't be used." diff --git a/src/modules/inject.py b/src/modules/inject.py index 957b749..69a6f5f 100644 --- a/src/modules/inject.py +++ b/src/modules/inject.py @@ -1,21 +1,22 @@ -from typing import NamedTuple, Union +from typing import NamedTuple, Optional, Union from bs4 import BeautifulSoup, Tag import modules.constants as C import modules.userscript as userscript from modules.userscript import Userscript, document_end, document_idle -from modules.utilities import fromOptional, idem, stripIndentation +from modules.utilities import fromOptional, idem, isSomething, stripIndentation class Options(NamedTuple): inline: bool - nonce: str + nonce: Optional[str] def inject(script: Userscript, soup: BeautifulSoup, options: Options) -> Union[BeautifulSoup, Exception]: useInline = options.inline or script.downloadURL is None tag = soup.new_tag("script") - tag["nonce"] = options.nonce # Used to bypass CSP for inline-injected userscripts. + if isSomething(options.nonce): + tag["nonce"] = options.nonce # Used to bypass CSP for inline-injected userscripts. tag[C.ATTRIBUTE_UP_VERSION] = C.VERSION withLoadListenerIfRunAtIdle = userscript.withEventListener("load") if script.runAt == document_idle else idem withNoframesIfNoframes = userscript.withNoframes if script.noframes else idem From 69013b810fc29c6fbd1270e40c60b742147cbfd3 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 21 Mar 2021 20:18:12 +0100 Subject: [PATCH 6/7] Document --bypass-csp --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index af71fd5..3ceb79f 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,15 @@ docker run -t --rm --name userscript-proxy -p 8080:8080 alling/userscript-proxy # flags to `docker run` flags to Userscript Proxy ``` +## `--bypass-csp ALLOW` + +Bypass host site's Content Security Policy (if any) to allow userscripts to run properly. +If `ALLOW` is `script`, the CSP is bypassed only for the userscript itself. +Use `nothing` to never bypass any CSP (meaning userscripts won't work at all on some sites). +Use `everything` to allow everything, which may be necessary if the userscript injects CSS, images etc. +Note that the latter completely disables any CSP from every host site into which a userscript is injected. +Defaults to `script`. + ## `--inline`, `-i` Always inject scripts inline (``), never linked (``). From 65daf9588e64ea02e0b4f5bd5de27508ed93b967 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 21 Mar 2021 20:35:27 +0100 Subject: [PATCH 7/7] Update readme table of contents --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ceb79f..5ce5ef3 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Both HTTP and HTTPS are supported. * [Data usage](#data-usage) * [Userscript compatibility](#userscript-compatibility) * [Options](#options) + * [--bypass-csp ALLOW](#--bypass-csp-allow) * [--inline, -i](#--inline--i) * [--list-injected, -l](#--list-injected--l) * [--no-default-rules](#--no-default-rules) @@ -35,7 +36,7 @@ Both HTTP and HTTPS are supported. * [--userscripts-dir DIR, -u DIR](#--userscripts-dir-dir--u-dir) * [Contribute](#contribute) - +