From df4dda1a95ecfee940f4e27cd59ec0e52c5a525a Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 21 Mar 2021 19:32:33 +0100 Subject: [PATCH 01/62] Add FUNDING.yml (Ko-fi) --- FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 FUNDING.yml diff --git a/FUNDING.yml b/FUNDING.yml new file mode 100644 index 0000000..b669b04 --- /dev/null +++ b/FUNDING.yml @@ -0,0 +1 @@ +ko_fi: alling From 647a4ca37f4c1e7702d85897aa452f9abdba03fb Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 21 Mar 2021 20:46:19 +0100 Subject: [PATCH 02/62] Add feature to bypass Content Security Policy (#7) As pointed out by @deatondg in #6, some sites have a CSP that prevents userscripts from running properly. This PR makes it possible to either bypass the CSP specifically for any userscripts that are injected (inline or not) or disable the CSP altogether whenever a userscript is injected. The latter is often necessary because userscripts tend to inject at least one resource into the page, be it an external image or just some inline CSS. Resolves #6. Co-authored-by: deatondg --- README.md | 12 +++++++++++- src/injector.py | 33 +++++++++++++++++++++++++++++---- src/launcher.py | 2 ++ src/modules/argparser.py | 8 ++++++++ src/modules/arguments.py | 9 +++++++++ src/modules/csp.py | 40 ++++++++++++++++++++++++++++++++++++++++ src/modules/inject.py | 7 +++++-- 7 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 src/modules/csp.py diff --git a/README.md b/README.md index af71fd5..5ce5ef3 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Both HTTP and HTTPS are supported. * [Data usage](#data-usage) * [Userscript compatibility](#userscript-compatibility) * [Options](#options) + * [--bypass-csp ALLOW](#--bypass-csp-allow) * [--inline, -i](#--inline--i) * [--list-injected, -l](#--list-injected--l) * [--no-default-rules](#--no-default-rules) @@ -35,7 +36,7 @@ Both HTTP and HTTPS are supported. * [--userscripts-dir DIR, -u DIR](#--userscripts-dir-dir--u-dir) * [Contribute](#contribute) - + @@ -287,6 +288,15 @@ docker run -t --rm --name userscript-proxy -p 8080:8080 alling/userscript-proxy # flags to `docker run` flags to Userscript Proxy ``` +## `--bypass-csp ALLOW` + +Bypass host site's Content Security Policy (if any) to allow userscripts to run properly. +If `ALLOW` is `script`, the CSP is bypassed only for the userscript itself. +Use `nothing` to never bypass any CSP (meaning userscripts won't work at all on some sites). +Use `everything` to allow everything, which may be necessary if the userscript injects CSS, images etc. +Note that the latter completely disables any CSP from every host site into which a userscript is injected. +Defaults to `script`. + ## `--inline`, `-i` Always inject scripts inline (``), never linked (``). diff --git a/src/injector.py b/src/injector.py index f3fc8cb..8db983c 100644 --- a/src/injector.py +++ b/src/injector.py @@ -9,6 +9,7 @@ import modules.arguments as A import modules.constants as C +import modules.csp as csp import modules.inject as inject import modules.inline as inline import modules.metadata as metadata @@ -139,6 +140,7 @@ def load(self, loader): loader.add_option(sanitize(A.inline), bool, False, A.inline_help) loader.add_option(sanitize(A.no_default_userscripts), bool, False, A.no_default_userscripts_help) loader.add_option(sanitize(A.list_injected), bool, False, A.list_injected_help) + loader.add_option(sanitize(A.bypass_csp), Optional[str], A.bypass_csp_default, A.bypass_csp_help) loader.add_option(sanitize(A.userscripts_dir), Optional[str], A.userscripts_dir_default, A.userscripts_dir_help) loader.add_option(sanitize(A.query_param_to_disable), str, A.query_param_to_disable_default, A.query_param_to_disable_help) @@ -167,7 +169,7 @@ def response(self, flow: http.HTTPFlow): if CONTENT_TYPE in response.headers: if any(map(lambda t: t in response.headers[CONTENT_TYPE], RELEVANT_CONTENT_TYPES)): # Response is a web page; proceed. - insertedScripts: List[str] = [] + injections: List[csp.Injection] = [] soup = BeautifulSoup( response.content, HTML_PARSER, @@ -185,23 +187,31 @@ def response(self, flow: http.HTTPFlow): logError(unsafeSequencesMessage(script)) continue logInfo(f"""Injecting {script.name}{"" if script.version is None else " " + C.VERSION_PREFIX + script.version} into {requestURL} ({"inline" if useInline else "linked"}) ...""") + shouldUseNonce = useInline and option(A.bypass_csp) == A.bypass_csp_script # If not inline, then URL is used for bypassing; if bypass for nothing or everything, then the nonce would have no effect anyway. + nonce = csp.generateNonce() if shouldUseNonce else None result = inject.inject(script, soup, inject.Options( inline = option(A.inline), + nonce = nonce )) if type(result) is BeautifulSoup: soup = result - insertedScripts.append(script.name + ("" if script.version is None else " " + T.stringifyVersion(script.version))) + injections.append(csp.Injection( + userscript = script, + nonce = nonce, + )) else: logError("Injection failed due to the following error:") logError(str(result)) + handleContentSecurityPolicy(response, injections) index_DTD: Optional[int] = indexOfDTD(soup) # Insert information comment: if option(A.list_injected): + namesOfInjectedScripts = [ i.userscript.name + ("" if i.userscript.version is None else " " + T.stringifyVersion(i.userscript.version)) for i in injections ] soup.insert(0 if index_DTD is None else 1+index_DTD, Comment( HTML_INFO_COMMENT_PREFIX + ( - "No matching userscripts for this URL." if insertedScripts == [] - else "These scripts were inserted:\n" + bulletList(insertedScripts) + "No matching userscripts for this URL." if namesOfInjectedScripts == [] + else "These scripts were inserted:\n" + bulletList(namesOfInjectedScripts) ) + "\n" )) # Serialize and encode: @@ -211,4 +221,19 @@ def response(self, flow: http.HTTPFlow): ) +def handleContentSecurityPolicy(response: http.HTTPFlow.response, injections: List[csp.Injection]): + # If there is a CSP header, we may need to modify it for the userscript(s) to work. + ContentSecurityPolicy = "Content-Security-Policy" + if ContentSecurityPolicy in response.headers: + bypassCspValue = option(A.bypass_csp) + if bypassCspValue == A.bypass_csp_script: + logInfo(f"Bypassing host site's Content Security Policy for userscripts only (not any resources injected _by_ userscripts, such as stylesheets and images). Try `{flag(A.bypass_csp)} {A.bypass_csp_everything}` if something does not work properly.") + response.headers[ContentSecurityPolicy] = csp.headerWithScriptsAllowed(response.headers[ContentSecurityPolicy], injections) + elif bypassCspValue == A.bypass_csp_everything: + logInfo(f"Bypassing host site's Content Security Policy altogether due to `{flag(A.bypass_csp)} {A.bypass_csp_everything}`.") + del response.headers[ContentSecurityPolicy] + else: + logWarning(f"Host site has a Content Security Policy. Try the {flag(A.bypass_csp)} flag if userscripts don't work properly.") + + addons = [ UserscriptInjector() ] diff --git a/src/launcher.py b/src/launcher.py index f9804a8..f868cb8 100755 --- a/src/launcher.py +++ b/src/launcher.py @@ -55,6 +55,7 @@ def checkThatUserscriptsDirectoryExistsIfSpecified(directory: str): useTransparent = args.transparent useFiltering = useCustomFiltering or useDefaultRules useIntercept = args.intercept is True + bypassCsp = args.bypass_csp userscriptsDirectory = args.userscripts_dir checkThatUserscriptsDirectoryExistsIfSpecified(userscriptsDirectory) def ruleFilesContent_default(): @@ -101,6 +102,7 @@ def ruleFilesContent_custom(): "--set", f"""{sanitize(A.inline)}={str(args.inline).lower()}""", "--set", f"""{sanitize(A.list_injected)}={str(args.list_injected).lower()}""", "--set", f"""{sanitize(A.no_default_userscripts)}={str(args.no_default_userscripts).lower()}""", + "--set", "" if bypassCsp is None else f"""{sanitize(A.bypass_csp)}={bypassCsp}""", "--set", "" if userscriptsDirectory is None else f"""{sanitize(A.userscripts_dir)}={userscriptsDirectory}""", "--set", f"""{sanitize(A.query_param_to_disable)}={args.query_param_to_disable}""", # Empty string breaks the argument chain: diff --git a/src/modules/argparser.py b/src/modules/argparser.py index 2c34f83..4a7f525 100644 --- a/src/modules/argparser.py +++ b/src/modules/argparser.py @@ -6,6 +6,14 @@ def getArgparser(): argparser = ArgumentParser(description=T.description) + argparser.add_argument( + flag(A.bypass_csp), + type=str, + metavar=A.metavar_allow, + choices=A.bypass_csp_values, + default=A.bypass_csp_default, + help=A.bypass_csp_help, + ) argparser.add_argument( flag(A.intercept), action="store_true", diff --git a/src/modules/arguments.py b/src/modules/arguments.py index 7a3fe6e..022c2ac 100644 --- a/src/modules/arguments.py +++ b/src/modules/arguments.py @@ -4,9 +4,18 @@ metavar_file = "FILE" metavar_dir = "DIR" metavar_param = "PARAM" +metavar_allow = "ALLOW" RULES = "rules" +bypass_csp = "bypass-csp" +bypass_csp_nothing = "nothing" +bypass_csp_script = "script" +bypass_csp_everything = "everything" +bypass_csp_default = bypass_csp_script +bypass_csp_values = { bypass_csp_nothing, bypass_csp_script, bypass_csp_everything } +bypass_csp_help = f"Bypass host site's Content Security Policy to allow userscripts to run properly. If {metavar_allow} is '{bypass_csp_script}', the CSP is bypassed only for the userscript itself. Use '{bypass_csp_everything}' to allow everything, which may be necessary if the userscript injects CSS, images etc. Note that the latter completely disables any CSP from every host site into which a userscript is injected. Default: '{bypass_csp_default}'." + inline = "inline" inline_short = "i" inline_help = "Always insert userscripts inline, never linked" diff --git a/src/modules/csp.py b/src/modules/csp.py new file mode 100644 index 0000000..9a6f41f --- /dev/null +++ b/src/modules/csp.py @@ -0,0 +1,40 @@ +import secrets +from typing import List, NamedTuple, Optional + +from modules.userscript import Userscript +from modules.utilities import isSomething + +# Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy + + +class Injection(NamedTuple): + userscript: Userscript + nonce: Optional[str] + + +def headerWithScriptsAllowed(cspHeaderValue: str, injections: List[Injection]) -> str: + # Example CSP header: + # + # Content-Security-Policy: default-src 'self'; frame-src 'self'; img-src https:; connect-src 'self' + # + cspKeyValuePairs = [ directive.strip().split(" ", 1) for directive in cspHeaderValue.split(';') ] + cspDict = { key: value for key, value in cspKeyValuePairs } + if "script-src" not in cspDict: + # Browsers fall back to default-src if there is no script-src. + # Since there was no script-src directive and we are adding one, we include the default-src (if present) in it to avoid breaking the site's effective CSP. + cspDict["script-src"] = cspDict["default-src"] if "default-src" in cspDict else "" + sourcesToAllow = [ source(i) for i in injections ] + cspDict["script-src"] += " " + " ".join(sourcesToAllow) + return '; '.join([ f'{key} {value}' for key, value in cspDict.items() ]) + + +def source(injection: Injection) -> str: + if isSomething(injection.nonce): + return f"'nonce-{injection.nonce}'" + else: + # MDN about host (i.e. download URL) sources: "Unlike other values below, single quotes shouldn't be used." + return injection.userscript.downloadURL + + +def generateNonce(): + return secrets.token_hex() # If no argument is passed, "a reasonable default is used" for the number of bytes. diff --git a/src/modules/inject.py b/src/modules/inject.py index b6de896..69a6f5f 100644 --- a/src/modules/inject.py +++ b/src/modules/inject.py @@ -1,19 +1,22 @@ -from typing import NamedTuple, Union +from typing import NamedTuple, Optional, Union from bs4 import BeautifulSoup, Tag import modules.constants as C import modules.userscript as userscript from modules.userscript import Userscript, document_end, document_idle -from modules.utilities import fromOptional, idem, stripIndentation +from modules.utilities import fromOptional, idem, isSomething, stripIndentation class Options(NamedTuple): inline: bool + nonce: Optional[str] def inject(script: Userscript, soup: BeautifulSoup, options: Options) -> Union[BeautifulSoup, Exception]: useInline = options.inline or script.downloadURL is None tag = soup.new_tag("script") + if isSomething(options.nonce): + tag["nonce"] = options.nonce # Used to bypass CSP for inline-injected userscripts. tag[C.ATTRIBUTE_UP_VERSION] = C.VERSION withLoadListenerIfRunAtIdle = userscript.withEventListener("load") if script.runAt == document_idle else idem withNoframesIfNoframes = userscript.withNoframes if script.noframes else idem From 048067f5bda9a7f2e4efc5b96de84052ff1fa7c0 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Tue, 23 Mar 2021 20:35:21 +0100 Subject: [PATCH 03/62] v1.1.0 (#9) * Add feature to bypass Content Security Policy (#7) --- src/modules/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/constants.py b/src/modules/constants.py index 186e765..264c6a4 100644 --- a/src/modules/constants.py +++ b/src/modules/constants.py @@ -1,6 +1,6 @@ VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" -VERSION: str = "1.0.0" +VERSION: str = "1.1.0" ATTRIBUTE_UP_VERSION: str = "data-userscript-proxy-version" DEFAULT_RULES_DIR: str = "default-rules/" From 2900deec63aa02e6e0a649c36441b8f8dc761be4 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 21:14:07 +0200 Subject: [PATCH 04/62] Pin dependency versions Now when I returned to this project a few years later, of course it doesn't work anymore. `make start` successfully builds a Docker image, but it fails like this after printing the list of hosts that will be ignored: Traceback (most recent call last): File "/root/.local/bin/mitmdump", line 8, in sys.exit(mitmdump()) File "/root/.local/lib/python3.7/site-packages/mitmproxy/tools/_main.py", line 153, in mitmdump from mitmproxy.tools import dump File "/root/.local/lib/python3.7/site-packages/mitmproxy/tools/dump.py", line 1, in from mitmproxy import addons File "/root/.local/lib/python3.7/site-packages/mitmproxy/addons/__init__.py", line 12, in from mitmproxy.addons import onboarding File "/root/.local/lib/python3.7/site-packages/mitmproxy/addons/onboarding.py", line 2, in from mitmproxy.addons.onboardingapp import app File "/root/.local/lib/python3.7/site-packages/mitmproxy/addons/onboardingapp/__init__.py", line 3, in from flask import Flask, render_template File "/root/.local/lib/python3.7/site-packages/flask/__init__.py", line 14, in from jinja2 import escape File "/root/.local/lib/python3.7/site-packages/jinja2/__init__.py", line 12, in from .environment import Environment File "/root/.local/lib/python3.7/site-packages/jinja2/environment.py", line 25, in from .defaults import BLOCK_END_STRING File "/root/.local/lib/python3.7/site-packages/jinja2/defaults.py", line 3, in from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 File "/root/.local/lib/python3.7/site-packages/jinja2/filters.py", line 13, in from markupsafe import soft_unicode ImportError: cannot import name 'soft_unicode' from 'markupsafe' (/root/.local/lib/python3.7/site-packages/markupsafe/__init__.py) I asked ChatGPT what this was about and got this explanation: > The error you're seeing is due to a **breaking change in the `markupsafe` package**, specifically: > > * As of `markupsafe >= 2.1.0`, the `soft_unicode` function has been **removed**. > > * Older packages like `Jinja2` (or any other dependency that hasn't been updated to match) **still try to import** `soft_unicode`. And yeah, [indeed](https://github.com/pallets/markupsafe/issues/282). ChatGPT proposed three different solutions: * Pin `markupsafe` to a compatible version like `<2.1.0`. * Upgrade `Jinja2`, `Flask`, or any other package that uses `markupsafe`. * Freeze a working `requirements.txt` from 2021. I chose to go for the latter, which I acheived like this: ```bash docker run -it --rm --name userscript-proxy -p 8080:8080 --entrypoint /bin/bash alling/userscript-proxy:1.1.0 # Inside the container: python --version # Printed "Python 3.7.10" pip freeze > /tmp/requirements.txt # Ctrl + D to exit, then: docker cp userscript-proxy:/tmp/requirements.txt . ``` Also, from now on, we won't be hard-wrapping commit messages at 72 characters anymore. --- Dockerfile | 2 +- requirements.txt | 44 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf99265..d3367ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7-slim AS base +FROM python:3.7.10-slim AS base FROM base AS builder diff --git a/requirements.txt b/requirements.txt index 196f515..f73c0f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,40 @@ -mitmproxy -beautifulsoup4 -urlmatch -lxml +asgiref==3.3.1 +beautifulsoup4==4.9.3 +blinker==1.4 +Brotli==1.0.9 +certifi==2020.12.5 +cffi==1.14.5 +click==7.1.2 +cryptography==3.2.1 +Flask==1.1.2 +h11==0.12.0 +h2==4.0.0 +hpack==4.0.0 +hyperframe==6.0.0 +itsdangerous==1.1.0 +Jinja2==2.11.3 +kaitaistruct==0.9 +ldap3==2.8.1 +lxml==4.6.3 +MarkupSafe==1.1.1 +mitmproxy==5.3.0 +msgpack==1.0.2 +passlib==1.7.4 +protobuf==3.13.0 +publicsuffix2==2.20191221 +pyasn1==0.4.8 +pycparser==2.20 +pyOpenSSL==19.1.0 +pyparsing==2.4.7 +pyperclip==1.8.2 +ruamel.yaml==0.16.13 +ruamel.yaml.clib==0.2.2 +six==1.15.0 +sortedcontainers==2.2.2 +soupsieve==2.2.1 +tornado==6.1 +urlmatch==1.0.1 +urwid==2.1.2 +Werkzeug==1.0.1 +wsproto==0.15.0 +zstandard==0.14.1 From 68f51c7a42bd8fe0659a878c1538976e46e16b35 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 23:01:09 +0200 Subject: [PATCH 05/62] Remove readme TOC and 'docs' Make target This commit essentially reverts 0202b917acdddd82d14f0e706e26a0b07c5fc6af. Maintaining the table of contents involves manual steps and s error-prone. For example, now that I ran `make docs`, it just erased the entire TOC in `README.md`. Given that GitHub [generates] a TOC automatically nowadays, removing our homegrown setup is an easy choice. [generates]: https://github.blog/changelog/2021-04-13-table-of-contents-support-in-markdown-files/ --- .gitignore | 3 --- Makefile | 15 --------------- README.md | 34 ---------------------------------- 3 files changed, 52 deletions(-) diff --git a/.gitignore b/.gitignore index 870917d..7bbc71c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -# Project-specific stuff: -gh-md-toc - # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Makefile b/Makefile index 7f55c74..bb75a06 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,3 @@ -TOC_FILE = gh-md-toc -TOC_HASH = 042fc595336c3a39f82b1edbafdf2afd2503d9930d192fcfda757aa65522c14c -TOC_URL = https://raw.githubusercontent.com/ekalinin/github-markdown-toc/56f7c5939e2119bed86291ddba9fb6c2ee61fb09/gh-md-toc - DEFAULT_TAG = latest TAG ?= $(DEFAULT_TAG) @@ -16,17 +12,6 @@ CA_DIR = /root/.mitmproxy .PHONY : all all: image -docs: - wget -O $(TOC_FILE) $(TOC_URL) -# Check that the file hasn't been tampered with: - echo "$(TOC_HASH) $(TOC_FILE)" | sha256sum -c - chmod +x $(TOC_FILE) -# Generate and insert TOC: - ./$(TOC_FILE) --insert README.md -# Remove files created by gh-md-toc: - rm README.md.orig.* - rm README.md.toc.* - image: docker build -t $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) . diff --git a/README.md b/README.md index 5ce5ef3..fff7eed 100644 --- a/README.md +++ b/README.md @@ -6,40 +6,6 @@ No jailbreak/root required. Userscript Proxy is built around [mitmproxy][mitmproxy] and acts as a MITM, injecting userscripts into web pages as they flow through it. Both HTTP and HTTPS are supported. - - * [Userscript Proxy](#userscript-proxy) - * [Getting started](#getting-started) - * [Security notice](#security-notice) - * [Starting the proxy](#starting-the-proxy) - * [On a mobile device](#on-a-mobile-device) - * [HTTPS](#https) - * [Android](#android) - * [iOS](#ios) - * [Deploying userscripts](#deploying-userscripts) - * [Apps with certificate pinning](#apps-with-certificate-pinning) - * [Basic pattern](#basic-pattern) - * [Examples](#examples) - * [Regular expression](#regular-expression) - * [Examples](#examples-1) - * [Data usage](#data-usage) - * [Userscript compatibility](#userscript-compatibility) - * [Options](#options) - * [--bypass-csp ALLOW](#--bypass-csp-allow) - * [--inline, -i](#--inline--i) - * [--list-injected, -l](#--list-injected--l) - * [--no-default-rules](#--no-default-rules) - * [--no-default-userscripts](#--no-default-userscripts) - * [--port PORT, -p PORT](#--port-port--p-port) - * [--query-param-to-disable PARAM, -q PARAM](#--query-param-to-disable-param--q-param) - * [--rules FILE](#--rules-file) - * [--transparent, -t](#--transparent--t) - * [--userscripts-dir DIR, -u DIR](#--userscripts-dir-dir--u-dir) - * [Contribute](#contribute) - - - - - # Getting started From e92411675fb7ae10d49f0962096d38255dafaa6f Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 23:23:41 +0200 Subject: [PATCH 06/62] Readme: Make documentation a bit less verbose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ’‘ `git show --color-words='This should .+|\w+|.'` --- README.md | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index fff7eed..bb0b04e 100644 --- a/README.md +++ b/README.md @@ -21,11 +21,6 @@ Make sure you understand these security aspects before using Userscript Proxy: ## Starting the proxy 1. Make sure you have [Docker](https://www.docker.com) installed. - This should work: - - ``` - docker --version - ``` 1. Start Userscript Proxy: @@ -59,14 +54,12 @@ Make sure you understand these security aspects before using Userscript Proxy: This is usually something like `192.168.1.67`. You can typically [find it](https://google.com/search?q=find+local+IP+address) by running `ip a`, `ifconfig` or `ipconfig` depending on your operating system. - If your local IP address is `192.168.1.67`, and the proxy is running (see above), this should work: + This should work on any computer within the LAN: ``` curl --proxy 192.168.1.67:8080 http://example.com ``` -1. Your mobile device needs to be on the same LAN as your proxy, so make sure it's connected to your Wi-Fi. - 1. On your mobile device, go to the settings for the currently active Wi-Fi connection. Find the proxy settings, select **Manual proxy** or similar, and set `192.168.1.67` with port `8080`. @@ -130,7 +123,7 @@ Otherwise, read on. ## Deploying userscripts Userscript Proxy comes with one single userscript, useful only for testing that the proxy is up and running. -To use userscripts you've downloaded or written yourself, you need to tell Userscript Proxy where they are. +To use userscripts you've downloaded or written yourself: 1. You need the **absolute path** to a directory containing your userscripts. This could be something like `/home/alling/userscripts`. @@ -223,7 +216,7 @@ A userscript is injected by reference if and only if it has a specified `@downlo (This can be overridden using the `--inline` flag, in which case all userscripts are injected inline.) Userscripts are injected into _every_ response from a matching URL, and the size of a userscript can be anything from a few hundred bytes for the most basic ones to hundreds of kilobytes in extreme cases, so there are _massive_ data usage reductions to be gained from making the userscript accessible by URL and including a `@downloadURL`. -If the `@downloadURL` approach is not possible, for one reason or the other, it is a good idea to be aware of this issue, and to take appropriate action such as [minifying][minification] userscripts and adding suitable ignore rules. +If the `@downloadURL` approach is not possible, for one reason or the other, it is a good idea to be aware of this issue, and to take appropriate action such as [minifying] userscripts and adding suitable ignore rules. # Userscript compatibility @@ -340,7 +333,7 @@ make start [mitmproxy]: https://mitmproxy.org -[minification]: https://en.wikipedia.org/wiki/Minification_(programming) +[minifying]: https://en.wikipedia.org/wiki/Minification_(programming) [metadata]: https://wiki.greasespot.net/Metadata_Block [transparent-mode]: https://docs.mitmproxy.org/stable/concepts-modes/#transparent-proxy [gm-api]: https://wiki.greasespot.net/GM.getValue From fe0827984315c9cc6f6724257affbcdf32955549 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 23:26:47 +0200 Subject: [PATCH 07/62] Readme: Clarify custom userscripts/rules documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two explanatory bullet points are removed because I feel like the addition of `my-` makes them superfluous. πŸ’‘ `git show --color-words=.` --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bb0b04e..0d880d5 100644 --- a/README.md +++ b/README.md @@ -131,12 +131,9 @@ To use userscripts you've downloaded or written yourself: 1. Run Userscript Proxy like this: ``` - docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" -v "/home/alling/userscripts:/userscripts" alling/userscript-proxy --userscripts-dir "/userscripts" + docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" -v "/home/alling/userscripts:/my-userscripts" alling/userscript-proxy --userscripts-dir "/my-userscripts" ``` - * `-v "/home/alling/userscripts:/userscripts"` mounts your userscripts directory at `/userscripts` inside the Docker container. - * `--userscripts-dir "/userscripts"` tells Userscript Proxy to read userscripts from `/userscripts`. - # Apps with certificate pinning @@ -157,11 +154,11 @@ Examples: * Take ignore rules from `/home/alling/rules/ignore.txt`: ```bash - docker run -t --rm -v "/home/alling/rules:/rules" alling/userscript-proxy --rules "/rules/ignore.txt" + docker run -t --rm -v "/home/alling/rules:/my-rules" alling/userscript-proxy --rules "/my-rules/ignore.txt" ``` * Take intercept rules from all `.txt` files in the `/home/alling/rules` directory whose names start with `foo`: ```bash - docker run -t --rm -v "/home/alling/rules:/rules" alling/userscript-proxy --rules "/rules/foo*.txt" --intercept + docker run -t --rm -v "/home/alling/rules:/my-rules" alling/userscript-proxy --rules "/my-rules/foo*.txt" --intercept ``` Rules can be specified in two ways: From 18801ca7a736ae70300f59ef8b3f50ea086dbd17 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 23:29:33 +0200 Subject: [PATCH 08/62] Readme: Simplify HTTPS documentation Most likely, users always want HTTPS support, so why act like it's something optional that they _might_ want to add later? This commit makes the very first suggested command launch Userscript Proxy with persistent certificates, and also removes some extraneous details about HTTPS and certificates that users probably aren't interested in. --- README.md | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 0d880d5..e3d54df 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Make sure you understand these security aspects before using Userscript Proxy: 1. Start Userscript Proxy: ``` - docker run -t --rm --name userscript-proxy -p 8080:8080 alling/userscript-proxy + docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" alling/userscript-proxy ``` When you see _Proxy server listening at http://*:8080_, the proxy is up and running. @@ -68,29 +68,7 @@ Make sure you understand these security aspects before using Userscript Proxy: ## HTTPS -When you've set up Userscript Proxy on your mobile device as described above, you'll notice that you can't visit sites via HTTPS anymore. -This is because your device thinks you're being [MITM'd](https://en.wikipedia.org/wiki/Man-in-the-middle_attack) (which, technically, you are – by yourself). - -To make HTTPS connections work, you need to tell your device that it should trust your proxy. -This is accomplished by installing a certificate. - -**In general, installing a certificate might pose a security risk. If you don't trust me and mitmproxy, stop here.** -Otherwise, read on. - -1. Stop the proxy by pressing `Ctrl` + `C` in the terminal where it's running. - Then start it again, this time with the `-v` flag as shown below: - - ``` - docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" alling/userscript-proxy - ``` - - This creates a new Docker volume and mounts it at `/root/.mitmproxy`, where mitmproxy stores its certificate authority files. - This is necessary so that you can restart the proxy later without having to perform all these steps again. - - In this example, `mitmproxy-ca` is the name of the new Docker volume. - You can choose any name you want, as long as it's not already in use. - -1. Make sure your mobile device is configured to use the proxy as decribed above. +To make HTTPS work, you need to make your device trust your proxy by installing a certificate generated by mitmproxy. 1. On your mobile device, go to [http://mitm.it](http://mitm.it). You should see icons for Apple, Windows, Android, etc. @@ -118,8 +96,6 @@ Otherwise, read on. 1. Under _Enable full trust for root certificates_, enable **mitmproxy**, confirming the action if prompted. -1. You should now be able to browse via HTTPS as usual. - ## Deploying userscripts Userscript Proxy comes with one single userscript, useful only for testing that the proxy is up and running. From e0e9aa3f7a9f847fec7d8b60f0af9549dbffca6a Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 23:39:57 +0200 Subject: [PATCH 09/62] Readme: Improve local-IP-address documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit About the changes in this commit: 1. It doesn't help the user to know that the command they should use depends on their OS. They'll figure it out anyway. 2. I feel like the `curl` command belongs in some kind of troubleshooting section, rather than the happy path. Also, it can give the impression that Userscript Proxy will always be running at 192.168.1.67. 3. Adding "e.g." should hopefully make it more obvious that 192.168.1.67 is just an example. πŸ’‘ `git show --color-words='curl.+com|.'` --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e3d54df..18353ac 100644 --- a/README.md +++ b/README.md @@ -52,16 +52,10 @@ Make sure you understand these security aspects before using Userscript Proxy: 1. You need to know the local IP address of the machine running Userscript Proxy (i.e. where you ran `docker run` above). This is usually something like `192.168.1.67`. - You can typically [find it](https://google.com/search?q=find+local+IP+address) by running `ip a`, `ifconfig` or `ipconfig` depending on your operating system. - - This should work on any computer within the LAN: - - ``` - curl --proxy 192.168.1.67:8080 http://example.com - ``` + You can typically [find it](https://google.com/search?q=find+local+IP+address) by running `ip a`, `ifconfig` or `ipconfig`. 1. On your mobile device, go to the settings for the currently active Wi-Fi connection. - Find the proxy settings, select **Manual proxy** or similar, and set `192.168.1.67` with port `8080`. + Find the proxy settings, select **Manual proxy** or similar, and set e.g. `192.168.1.67` with port `8080`. 1. Visit [`http://example.com`](http://example.com) on your mobile device. You should see the same green page as above. From ef6c883445042af49a940b3396ae22c775076fed Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 29 May 2025 00:20:10 +0200 Subject: [PATCH 10/62] Readme: Add Docker Compose file --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 18353ac..85e9caa 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,32 @@ Both HTTP and HTTPS are supported. # Getting started +If you're familiar with Userscript Proxy, you might want to use Docker Compose: + +```yaml +services: + userscript-proxy: + image: alling/userscript-proxy:v1.1.0 + container_name: userscript-proxy + command: + - --userscripts-dir + - /my-userscripts + - --rules + - /my-rules/ignore.txt + ports: + - "8080:8080" + volumes: + - mitmproxy-ca:/root/.mitmproxy + - /absolute/path/to/my/userscripts/:/my-userscripts # Modify the part before the ':'! + - /absolute/path/to/my/rules/:/my-rules # Modify the part before the ':'! + restart: always + +volumes: + mitmproxy-ca: +``` + +Otherwise, keep reading. + ## Security notice Make sure you understand these security aspects before using Userscript Proxy: From b485dbbff6b78ab61b79e01d7f0478a0830d842a Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 29 May 2025 00:32:11 +0200 Subject: [PATCH 11/62] Readme: Fix incorrect image tag in Compose file --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 85e9caa..38401ba 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ If you're familiar with Userscript Proxy, you might want to use Docker Compose: ```yaml services: userscript-proxy: - image: alling/userscript-proxy:v1.1.0 + image: alling/userscript-proxy:1.1.0 container_name: userscript-proxy command: - --userscripts-dir From 87a2a3cf07642dea8612af71409a1d170fe8589a Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 29 May 2025 11:58:17 +0200 Subject: [PATCH 12/62] Typecheck code with mypy in Dockerfile --- Dockerfile | 4 +++- requirements.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d3367ac..120681b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,17 +6,19 @@ COPY requirements.txt . # We're not going to run anything in the build container, so we'll suppress the script location warnings. RUN pip install --user --no-warn-script-location -r requirements.txt - FROM base WORKDIR /app COPY --from=builder /root/.local/lib /root/.local/lib COPY --from=builder /root/.local/bin/mitmdump /root/.local/bin/mitmdump +COPY --from=builder /root/.local/bin/mypy /root/.local/bin/mypy ENV PATH=/root/.local/bin:$PATH +COPY typecheck . COPY src src COPY default-rules default-rules COPY default-userscripts default-userscripts +RUN ./typecheck EXPOSE 8080 ENTRYPOINT [ "python", "-u", "src/launcher.py" ] diff --git a/requirements.txt b/requirements.txt index f73c0f8..cf24d23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ lxml==4.6.3 MarkupSafe==1.1.1 mitmproxy==5.3.0 msgpack==1.0.2 +mypy==1.4.1 passlib==1.7.4 protobuf==3.13.0 publicsuffix2==2.20191221 From 56f5a432a619ed018d27ebefc1bad89ee3541782 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 29 May 2025 12:01:15 +0200 Subject: [PATCH 13/62] Remove unused `isSomething` import It became unused in 4c55b27e299cb33d7895346de9f21bf07e3041ff. --- src/launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/launcher.py b/src/launcher.py index f868cb8..7eebc9b 100755 --- a/src/launcher.py +++ b/src/launcher.py @@ -12,7 +12,7 @@ import modules.ignore as ignore from modules.misc import sanitize import modules.text as T -from modules.utilities import flag, idem, isSomething, itemList +from modules.utilities import flag, idem, itemList FILENAME_INJECTOR: str = "injector.py" MATCH_NO_HOSTS = r"^$" From 3416bb510f51fe0779a9e9478d6cc7362c0b075e Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 29 May 2025 12:42:14 +0200 Subject: [PATCH 14/62] Remove `isSomething` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I feel like it just adds an unnecessary layer of indirection. πŸ’‘ `git show --color-words='\w+|.'` --- src/modules/csp.py | 3 +-- src/modules/inject.py | 4 ++-- src/modules/metadata.py | 6 +++--- src/modules/patterns.py | 8 ++++---- src/modules/userscript.py | 12 ++++++------ src/modules/utilities.py | 4 ---- 6 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/modules/csp.py b/src/modules/csp.py index 9a6f41f..230eb98 100644 --- a/src/modules/csp.py +++ b/src/modules/csp.py @@ -2,7 +2,6 @@ from typing import List, NamedTuple, Optional from modules.userscript import Userscript -from modules.utilities import isSomething # Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy @@ -29,7 +28,7 @@ def headerWithScriptsAllowed(cspHeaderValue: str, injections: List[Injection]) - def source(injection: Injection) -> str: - if isSomething(injection.nonce): + if injection.nonce is not None: return f"'nonce-{injection.nonce}'" else: # MDN about host (i.e. download URL) sources: "Unlike other values below, single quotes shouldn't be used." diff --git a/src/modules/inject.py b/src/modules/inject.py index 69a6f5f..4f639c1 100644 --- a/src/modules/inject.py +++ b/src/modules/inject.py @@ -5,7 +5,7 @@ import modules.constants as C import modules.userscript as userscript from modules.userscript import Userscript, document_end, document_idle -from modules.utilities import fromOptional, idem, isSomething, stripIndentation +from modules.utilities import fromOptional, idem, stripIndentation class Options(NamedTuple): inline: bool @@ -15,7 +15,7 @@ class Options(NamedTuple): def inject(script: Userscript, soup: BeautifulSoup, options: Options) -> Union[BeautifulSoup, Exception]: useInline = options.inline or script.downloadURL is None tag = soup.new_tag("script") - if isSomething(options.nonce): + if options.nonce is not None: tag["nonce"] = options.nonce # Used to bypass CSP for inline-injected userscripts. tag[C.ATTRIBUTE_UP_VERSION] = C.VERSION withLoadListenerIfRunAtIdle = userscript.withEventListener("load") if script.runAt == document_idle else idem diff --git a/src/modules/metadata.py b/src/modules/metadata.py index 1787bfb..d322cf1 100644 --- a/src/modules/metadata.py +++ b/src/modules/metadata.py @@ -3,7 +3,7 @@ from string import Template from typing import Callable, Iterable, Iterator, List, Match, NamedTuple, Optional, Pattern, Tuple, TypeVar, Union -from modules.utilities import first, isSomething, second +from modules.utilities import first, second class MetadataError(Exception): def __init__(self,*args,**kwargs): @@ -97,10 +97,10 @@ def tag(name: str) -> str: def isWhitespaceLine(s: str) -> bool: - return isSomething(re.compile(r"^\s*$").match(s)) + return re.compile(r"^\s*$").match(s) is not None def isCommentLine(s: str) -> bool: - return isSomething(re.compile(r"^\s*" + PREFIX_COMMENT + r".*$").match(s)) + return re.compile(r"^\s*" + PREFIX_COMMENT + r".*$").match(s) is not None def extract(userscriptContent: str) -> str: # raises MetadataError match_metadataBlock: Optional[Match] = REGEX_METADATA_BLOCK.search(userscriptContent) diff --git a/src/modules/patterns.py b/src/modules/patterns.py index ac02db5..cdb15d0 100644 --- a/src/modules/patterns.py +++ b/src/modules/patterns.py @@ -1,7 +1,7 @@ import re from typing import Match, Optional, Pattern -from modules.utilities import first, isSomething +from modules.utilities import first REGEX_MATCH_ALL = r"" REGEX_MATCH_SCHEME = r"\*|https?" @@ -34,15 +34,15 @@ def normalizeMatchPattern(pattern: str) -> str: def isMatchPattern(pattern: str) -> bool: - return isSomething(REGEX_MATCH_PATTERN.match(pattern)) + return REGEX_MATCH_PATTERN.match(pattern) is not None def isIncludePattern(pattern: str) -> bool: - return isSomething(REGEX_INCLUDE_PATTERN.match(pattern)) + return REGEX_INCLUDE_PATTERN.match(pattern) is not None def isIncludePattern_regex(pattern: str) -> bool: - return isSomething(re.compile(REGEX_INCLUDE_REGEX).match(pattern)) + return re.compile(REGEX_INCLUDE_REGEX).match(pattern) is not None def withoutSurroundingSlashes(s: str) -> str: diff --git a/src/modules/userscript.py b/src/modules/userscript.py index 00cf58a..7cbd328 100644 --- a/src/modules/userscript.py +++ b/src/modules/userscript.py @@ -9,7 +9,7 @@ import modules.metadata as metadata from modules.metadata import Metadata, Tag, Tag_boolean, Tag_string from modules.patterns import isIncludePattern, isMatchPattern, regexFromIncludePattern -from modules.utilities import compose2, isSomething, stripIndentation, strs +from modules.utilities import compose2, stripIndentation, strs class UserscriptError(Exception): def __init__(self,*args,**kwargs): @@ -84,7 +84,7 @@ def __init__(self,*args,**kwargs): unique = True, default = None, required = False, - predicate = lambda val: isSomething(REGEX_URL.match(val)), + predicate = lambda val: REGEX_URL.match(val) is not None, ) METADATA_TAGS: List[Tag] = [ @@ -133,14 +133,14 @@ def create(content: str) -> Userscript: valueOf = metadata.valueGetter_one(validMetadata) allValuesOf = metadata.valueGetter_all(validMetadata) includePatternRegexes: List[Pattern] = list(filter( - isSomething, + lambda x: x is not None, map( compose2(regexFromIncludePattern_safe, str), allValuesOf(tag_include) ) )) excludePatternRegexes: List[Pattern] = list(filter( - isSomething, + lambda x: x is not None, map( compose2(regexFromIncludePattern_safe, str), allValuesOf(tag_exclude) @@ -163,10 +163,10 @@ def create(content: str) -> Userscript: def applicableChecker(url: str) -> Callable[[Userscript], bool]: def isApplicable(userscript: Userscript) -> bool: for regex in userscript.excludePatternRegexes: - if isSomething(regex.search(url)): + if regex.search(url) is not None: return False for regex in userscript.includePatternRegexes: - if isSomething(regex.search(url)): + if regex.search(url) is not None: return True for pattern in userscript.matchPatterns: if urlmatch(pattern, url): diff --git a/src/modules/utilities.py b/src/modules/utilities.py index cb465f1..7a7628c 100644 --- a/src/modules/utilities.py +++ b/src/modules/utilities.py @@ -23,10 +23,6 @@ def second(tuple: Tuple[A, B]) -> B: return b -def isSomething(x: Optional[A]) -> bool: - return x is not None - - def strs(xs: Any) -> List[str]: return list(map(str, xs)) From bba90376ce0d0e46a22a3c55301609f664433e44 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 22 Jun 2025 16:04:55 +0200 Subject: [PATCH 15/62] Remove unused UserscriptError class As far as I can tell, it has never been used, in the sense that no instances have ever been constructed. See for example `git log -p -S UserscriptError`. --- src/injector.py | 6 +----- src/modules/userscript.py | 4 ---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/injector.py b/src/injector.py index 8db983c..9ce017c 100644 --- a/src/injector.py +++ b/src/injector.py @@ -17,7 +17,7 @@ from modules.requests import CONTENT_TYPE, containsQueryParam, inferEncoding import modules.text as T import modules.userscript as userscript -from modules.userscript import Userscript, UserscriptError +from modules.userscript import Userscript from modules.utilities import first, flag, fromOptional, itemList, second PATTERN_USERSCRIPT: str = "*.user.js" @@ -115,10 +115,6 @@ def loadUserscripts(directory: str) -> List[Userscript]: logError("Metadata error:") logError(str(err)) continue - except UserscriptError as err: - logError("Userscript error:") - logError(str(err)) - continue os.chdir(workingDirectory) # so mitmproxy does not unload the script logInfo("") logInfo(str(len(loadedUserscripts)) + " userscript(s) loaded:") diff --git a/src/modules/userscript.py b/src/modules/userscript.py index 7cbd328..7b60d87 100644 --- a/src/modules/userscript.py +++ b/src/modules/userscript.py @@ -11,10 +11,6 @@ from modules.patterns import isIncludePattern, isMatchPattern, regexFromIncludePattern from modules.utilities import compose2, stripIndentation, strs -class UserscriptError(Exception): - def __init__(self,*args,**kwargs): - Exception.__init__(self,*args,**kwargs) - REGEX_URL: Pattern = re.compile(r"^https?://") directive_name : str = "name" From 015f2ccc68420c06e4e7df133efc38b72864181f Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 13 Jul 2025 23:01:27 +0200 Subject: [PATCH 16/62] Remove support for typechecking individual files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It makes the typechecking script more complicated, and I never use it. πŸ’‘ `git show --ignore-all-space` --- typecheck | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/typecheck b/typecheck index a5bfa45..7adae6b 100755 --- a/typecheck +++ b/typecheck @@ -3,9 +3,5 @@ export MYPYPATH=/usr/local/lib/python3.6/dist-packages/ cd src -if [ "$1" == "" ]; then - mypy *.py --ignore-missing-imports --follow-imports skip - mypy modules/*.py --ignore-missing-imports --follow-imports skip -else - mypy $1 --ignore-missing-imports --follow-imports skip -fi +mypy *.py --ignore-missing-imports --follow-imports skip +mypy modules/*.py --ignore-missing-imports --follow-imports skip From a07173bc17ba596c20b22bd6a563bafbf3af8894 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 13 Jul 2025 23:03:41 +0200 Subject: [PATCH 17/62] Don't cd into `src/` in typechecking script It took me a while to realize that the `mypy` commands in the typechecking script didn't run in the repo root, but instead in `src/`. --- typecheck | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/typecheck b/typecheck index 7adae6b..870429d 100755 --- a/typecheck +++ b/typecheck @@ -2,6 +2,5 @@ export MYPYPATH=/usr/local/lib/python3.6/dist-packages/ -cd src -mypy *.py --ignore-missing-imports --follow-imports skip -mypy modules/*.py --ignore-missing-imports --follow-imports skip +mypy src/*.py --ignore-missing-imports --follow-imports skip +mypy src/modules/*.py --ignore-missing-imports --follow-imports skip From 3399b69e0f2e29e4a6d9f2a470f004736d6e7798 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 13 Jul 2025 23:11:49 +0200 Subject: [PATCH 18/62] Make typechecking fail on errors outside `src/modules/` Today, this change doesn't cause `make` to fail: ```diff --- a/src/launcher.py +++ b/src/launcher.py @@ -15,4 +15,5 @@ import modules.text as T from modules.utilities import flag, idem, itemList +foo: str = 5 FILENAME_INJECTOR: str = "injector.py" MATCH_NO_HOSTS = r"^$" ``` The first `mypy` command in the script fails, but without `-e`, the script just proceeds to the second `mypy` command, which succeeds. This commit makes the script fail as expected. --- typecheck | 2 ++ 1 file changed, 2 insertions(+) diff --git a/typecheck b/typecheck index 870429d..12be8aa 100755 --- a/typecheck +++ b/typecheck @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + export MYPYPATH=/usr/local/lib/python3.6/dist-packages/ mypy src/*.py --ignore-missing-imports --follow-imports skip From 0176bc7cfe611ff68e0ad1544260238f200fc193 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sat, 9 Aug 2025 10:45:34 +0200 Subject: [PATCH 19/62] Upgrade to Python 3.9 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 120681b..42793fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.10-slim AS base +FROM python:3.9.23-slim AS base FROM base AS builder From 377caf60f2f6afb595cce75faeb29603e06dba76 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sat, 9 Aug 2025 10:55:32 +0200 Subject: [PATCH 20/62] Update Compose file image version in release workflow The Compose file template was added in ef6c883445042af49a940b3396ae22c775076fed. --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bb75a06..294001e 100644 --- a/Makefile +++ b/Makefile @@ -27,9 +27,11 @@ ifeq "$(TAG)" "$(DEFAULT_TAG)" endif # Update in-app version: sed -i 's/^VERSION: str = "[^"]*"/VERSION: str = "$(TAG)"/' $(FILE_WITH_VERSION) +# Update readme version: + sed -i 's#image: alling/userscript-proxy:.*#image: alling/userscript-proxy:$(TAG)#' README.md docker build -t $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) . docker tag $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) $(DOCKER_USER)/$(DOCKER_REPO):$(DEFAULT_TAG) - git add $(FILE_WITH_VERSION) + git add $(FILE_WITH_VERSION) README.md git commit -m "v$(TAG)" git tag "v$(TAG)" From b0c4a262137272036e028782f53084e9a6062e1b Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sat, 9 Aug 2025 11:16:00 +0200 Subject: [PATCH 21/62] v1.1.1 --- README.md | 2 +- src/modules/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 38401ba..058c022 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ If you're familiar with Userscript Proxy, you might want to use Docker Compose: ```yaml services: userscript-proxy: - image: alling/userscript-proxy:1.1.0 + image: alling/userscript-proxy:1.1.1 container_name: userscript-proxy command: - --userscripts-dir diff --git a/src/modules/constants.py b/src/modules/constants.py index 264c6a4..c6970d7 100644 --- a/src/modules/constants.py +++ b/src/modules/constants.py @@ -1,6 +1,6 @@ VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" -VERSION: str = "1.1.0" +VERSION: str = "1.1.1" ATTRIBUTE_UP_VERSION: str = "data-userscript-proxy-version" DEFAULT_RULES_DIR: str = "default-rules/" From 31eb23c7247a064b2392f32966bb03281264ecda Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sat, 9 Aug 2025 11:22:41 +0200 Subject: [PATCH 22/62] Document how to push to Docker Hub The reason it's not multiple `echo` commands is that `make` prints each command, resulting in completely unreadable output. --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 294001e..14da995 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,7 @@ endif git add $(FILE_WITH_VERSION) README.md git commit -m "v$(TAG)" git tag "v$(TAG)" + echo "Run these commands to push to Docker Hub:\n\n docker push alling/userscript-proxy:$(TAG)\n docker push alling/userscript-proxy:$(DEFAULT_TAG)\n" start: image # The -t flag enables colored output: From 4d6a0fdd59120daf06b2418ac1f630f19de415de Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 13 Jul 2025 18:32:28 +0200 Subject: [PATCH 23/62] Use native `list` instead of `typing.List` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [PEP 585], included in Python 3.9, "enable[d] support for the generics syntax in all standard collections currently available in the `typing` module." This means that we can use `list` instead of importing `List` from the `typing` module. The intention is to make other analogous changes, such as replacing `typing.Tuple` with `tuple`, in future commits. [PEP 585]: https://peps.python.org/pep-0585/ πŸ’‘ `git show --color-words='\w+|.'` --- src/injector.py | 14 +++++++------- src/launcher.py | 7 +++---- src/modules/csp.py | 4 ++-- src/modules/ignore.py | 4 ++-- src/modules/inline.py | 6 +++--- src/modules/metadata.py | 16 ++++++++-------- src/modules/userscript.py | 16 ++++++++-------- src/modules/utilities.py | 4 ++-- 8 files changed, 35 insertions(+), 36 deletions(-) diff --git a/src/injector.py b/src/injector.py index 9ce017c..c01f185 100644 --- a/src/injector.py +++ b/src/injector.py @@ -2,7 +2,7 @@ import glob import os import shlex -from typing import Callable, Iterable, List, Optional, Tuple +from typing import Callable, Iterable, Optional, Tuple from bs4 import BeautifulSoup, Comment, Doctype from mitmproxy import ctx, http @@ -21,7 +21,7 @@ from modules.utilities import first, flag, fromOptional, itemList, second PATTERN_USERSCRIPT: str = "*.user.js" -RELEVANT_CONTENT_TYPES: List[str] = ["text/html", "application/xhtml+xml"] +RELEVANT_CONTENT_TYPES: list[str] = ["text/html", "application/xhtml+xml"] CHARSET_DEFAULT: str = "utf-8" TAB: str = " " LIST_ITEM_PREFIX: str = TAB + "β€’ " @@ -85,8 +85,8 @@ def option(key: str): return ctx.options.__getattr__(sanitize(key)) -def loadUserscripts(directory: str) -> List[Userscript]: - loadedUserscripts: List[Tuple[Userscript, str]] = [] +def loadUserscripts(directory: str) -> list[Userscript]: + loadedUserscripts: list[Tuple[Userscript, str]] = [] workingDirectory = os.getcwd() logInfo(f"""Looking recursively for userscripts ({PATTERN_USERSCRIPT}) in directory `{directory}` ...""") os.chdir(directory) @@ -129,7 +129,7 @@ def loadUserscripts(directory: str) -> List[Userscript]: class UserscriptInjector: def __init__(self): - self.userscripts: List[Userscript] = [] + self.userscripts: list[Userscript] = [] def load(self, loader): @@ -165,7 +165,7 @@ def response(self, flow: http.HTTPFlow): if CONTENT_TYPE in response.headers: if any(map(lambda t: t in response.headers[CONTENT_TYPE], RELEVANT_CONTENT_TYPES)): # Response is a web page; proceed. - injections: List[csp.Injection] = [] + injections: list[csp.Injection] = [] soup = BeautifulSoup( response.content, HTML_PARSER, @@ -217,7 +217,7 @@ def response(self, flow: http.HTTPFlow): ) -def handleContentSecurityPolicy(response: http.HTTPFlow.response, injections: List[csp.Injection]): +def handleContentSecurityPolicy(response: http.HTTPFlow.response, injections: list[csp.Injection]): # If there is a CSP header, we may need to modify it for the userscript(s) to work. ContentSecurityPolicy = "Content-Security-Policy" if ContentSecurityPolicy in response.headers: diff --git a/src/launcher.py b/src/launcher.py index 7eebc9b..7c0b21f 100755 --- a/src/launcher.py +++ b/src/launcher.py @@ -4,7 +4,6 @@ import os import shlex import subprocess -from typing import List from modules.argparser import getArgparser import modules.arguments as A @@ -22,7 +21,7 @@ def printInfo( useFiltering: bool, useIntercept: bool, useTransparent: bool, - filterRules: List[str], + filterRules: list[str], ): print() print("mitmproxy will be run in " + ("TRANSPARENT" if useTransparent else "REGULAR") + " mode.") @@ -62,7 +61,7 @@ def ruleFilesContent_default(): if useDefaultRules: print(f"Reading default {'intercept' if useIntercept else 'ignore'} rules ...") globPatternForDefaultRules = C.DEFAULT_INTERCEPT_RULES if useIntercept else C.DEFAULT_IGNORE_RULES - filenames: List[str] = [ shlex.quote(unsafeFilename) for unsafeFilename in glob.glob(globPatternForDefaultRules) ] + filenames: list[str] = [ shlex.quote(unsafeFilename) for unsafeFilename in glob.glob(globPatternForDefaultRules) ] acc = "" for filename in filenames: print("Reading " + filename + " ...") @@ -73,7 +72,7 @@ def ruleFilesContent_default(): def ruleFilesContent_custom(): if useCustomFiltering: print(f"Reading custom {'intercept' if useIntercept else 'ignore'} rules ({globPattern}) ...") - filenames: List[str] = [ shlex.quote(unsafeFilename) for unsafeFilename in glob.glob(globPattern) ] + filenames: list[str] = [ shlex.quote(unsafeFilename) for unsafeFilename in glob.glob(globPattern) ] acc = "" for filename in filenames: print("Reading " + filename + " ...") diff --git a/src/modules/csp.py b/src/modules/csp.py index 230eb98..36221f4 100644 --- a/src/modules/csp.py +++ b/src/modules/csp.py @@ -1,5 +1,5 @@ import secrets -from typing import List, NamedTuple, Optional +from typing import NamedTuple, Optional from modules.userscript import Userscript @@ -11,7 +11,7 @@ class Injection(NamedTuple): nonce: Optional[str] -def headerWithScriptsAllowed(cspHeaderValue: str, injections: List[Injection]) -> str: +def headerWithScriptsAllowed(cspHeaderValue: str, injections: list[Injection]) -> str: # Example CSP header: # # Content-Security-Policy: default-src 'self'; frame-src 'self'; img-src https:; connect-src 'self' diff --git a/src/modules/ignore.py b/src/modules/ignore.py index 7eb66b3..be02fcb 100644 --- a/src/modules/ignore.py +++ b/src/modules/ignore.py @@ -1,5 +1,5 @@ import re -from typing import List, Pattern +from typing import Pattern from modules.patterns import isIncludePattern_regex, regexify, withoutSurroundingSlashes @@ -8,7 +8,7 @@ PIPE: str = "|" REGEX_COMMENT: Pattern = re.compile(r"\#.*$") -def rulesIn(text: str) -> List[str]: +def rulesIn(text: str) -> list[str]: return list(filter( lambda s: s != "", map(withoutCommentAndTrimmed, text.splitlines()) diff --git a/src/modules/inline.py b/src/modules/inline.py index 58bda01..99166b8 100644 --- a/src/modules/inline.py +++ b/src/modules/inline.py @@ -1,9 +1,9 @@ import re -from typing import List, Pattern +from typing import Pattern # https://www.w3.org/TR/html/semantics-scripting.html#script-content-restrictions -DANGEROUS_SEQUENCES: List[str] = [ +DANGEROUS_SEQUENCES: list[str] = [ r"