From 354ada0d0b20f1a4f7604d698fdb368bd9f7972e Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 23 Mar 2020 20:21:42 +0100 Subject: [PATCH 001/137] Unify Dockerfiles Arguments can be passed like this: $ docker run userscript-proxy --transparent --- Dockerfile | 2 +- Dockerfile.transparent | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 Dockerfile.transparent diff --git a/Dockerfile b/Dockerfile index cee7271..b19081e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,4 +15,4 @@ COPY requirements.txt . RUN pip install -r requirements.txt COPY . ./ EXPOSE 8080 -CMD [ "./launcher.py", "--recursive", "--ignore", "ignore.txt" ] +ENTRYPOINT [ "./launcher.py" ] diff --git a/Dockerfile.transparent b/Dockerfile.transparent deleted file mode 100644 index 0d0d770..0000000 --- a/Dockerfile.transparent +++ /dev/null @@ -1,18 +0,0 @@ -FROM python:3.7-alpine as base - -RUN apk add -U --no-cache \ - gcc \ - build-base \ - linux-headers \ - ca-certificates \ - python3-dev \ - libffi-dev \ - openssl-dev \ - libxslt-dev -WORKDIR /app -RUN pip install mitmproxy -COPY requirements.txt . -RUN pip install -r requirements.txt -COPY . ./ -EXPOSE 8080 -CMD [ "./launcher.py", "--transparent", "--recursive" ] From bae28a82d2fa1ba0163b8db414bcf0c47443919c Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 23 Mar 2020 20:23:24 +0100 Subject: [PATCH 002/137] Fix output order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks to Andreas Lindhé. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b19081e..de3d2a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,4 +15,4 @@ COPY requirements.txt . RUN pip install -r requirements.txt COPY . ./ EXPOSE 8080 -ENTRYPOINT [ "./launcher.py" ] +ENTRYPOINT [ "python", "-u", "launcher.py" ] From 09b23d377a0167fe0ffb66da54b270bac5fb2ee3 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 23 Mar 2020 21:42:43 +0100 Subject: [PATCH 003/137] Simplify Dockerfile and decrease image size --- Dockerfile | 27 ++++++++++++++------------- requirements.txt | 1 + 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index de3d2a4..53b3300 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,19 @@ -FROM python:3.7-alpine as base +FROM python:3.7-slim AS base + +FROM base AS builder -RUN apk add -U --no-cache \ - gcc \ - build-base \ - linux-headers \ - ca-certificates \ - python3-dev \ - libffi-dev \ - openssl-dev \ - libxslt-dev -WORKDIR /app -RUN pip install mitmproxy COPY requirements.txt . -RUN pip install -r requirements.txt +# We're not going to run anything in the build container, so we'll suppress the script location warnings. +RUN pip install --user --no-warn-script-location -r requirements.txt + + +FROM base + +WORKDIR /app + +COPY --from=builder /root/.local /root/.local +ENV PATH=/root/.local/bin:$PATH COPY . ./ + EXPOSE 8080 ENTRYPOINT [ "python", "-u", "launcher.py" ] diff --git a/requirements.txt b/requirements.txt index f392d61..196f515 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +mitmproxy beautifulsoup4 urlmatch lxml From a0a6cf823379ff72d6a53368817f402bd7a4d296 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 23 Mar 2020 23:03:31 +0100 Subject: [PATCH 004/137] Clean up directory structure --- Dockerfile | 6 ++++-- ignore.txt => rules/ignore.txt | 0 intercept.txt => rules/intercept.txt | 0 injector.py => src/injector.py | 0 launcher.py => src/launcher.py | 2 +- {modules => src/modules}/constants.py | 0 {modules => src/modules}/ignore.py | 0 {modules => src/modules}/inject.py | 0 {modules => src/modules}/inline.py | 0 {modules => src/modules}/metadata.py | 0 {modules => src/modules}/misc.py | 0 {modules => src/modules}/patterns.py | 0 {modules => src/modules}/requests.py | 0 {modules => src/modules}/text.py | 0 {modules => src/modules}/userscript.py | 0 {modules => src/modules}/utilities.py | 0 16 files changed, 5 insertions(+), 3 deletions(-) rename ignore.txt => rules/ignore.txt (100%) rename intercept.txt => rules/intercept.txt (100%) rename injector.py => src/injector.py (100%) rename launcher.py => src/launcher.py (99%) rename {modules => src/modules}/constants.py (100%) rename {modules => src/modules}/ignore.py (100%) rename {modules => src/modules}/inject.py (100%) rename {modules => src/modules}/inline.py (100%) rename {modules => src/modules}/metadata.py (100%) rename {modules => src/modules}/misc.py (100%) rename {modules => src/modules}/patterns.py (100%) rename {modules => src/modules}/requests.py (100%) rename {modules => src/modules}/text.py (100%) rename {modules => src/modules}/userscript.py (100%) rename {modules => src/modules}/utilities.py (100%) diff --git a/Dockerfile b/Dockerfile index 53b3300..aa7fe26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,9 @@ WORKDIR /app COPY --from=builder /root/.local /root/.local ENV PATH=/root/.local/bin:$PATH -COPY . ./ +COPY src src +COPY rules rules +COPY userscripts userscripts EXPOSE 8080 -ENTRYPOINT [ "python", "-u", "launcher.py" ] +ENTRYPOINT [ "python", "-u", "src/launcher.py" ] diff --git a/ignore.txt b/rules/ignore.txt similarity index 100% rename from ignore.txt rename to rules/ignore.txt diff --git a/intercept.txt b/rules/intercept.txt similarity index 100% rename from intercept.txt rename to rules/intercept.txt diff --git a/injector.py b/src/injector.py similarity index 100% rename from injector.py rename to src/injector.py diff --git a/launcher.py b/src/launcher.py similarity index 99% rename from launcher.py rename to src/launcher.py index 5344302..5e21832 100755 --- a/launcher.py +++ b/src/launcher.py @@ -12,7 +12,7 @@ from argparse import ArgumentParser from functools import reduce -FILENAME_INJECTOR: str = "injector.py" +FILENAME_INJECTOR: str = "src/injector.py" MATCH_NO_HOSTS = r"^$" argparser = ArgumentParser(description=T.description) diff --git a/modules/constants.py b/src/modules/constants.py similarity index 100% rename from modules/constants.py rename to src/modules/constants.py diff --git a/modules/ignore.py b/src/modules/ignore.py similarity index 100% rename from modules/ignore.py rename to src/modules/ignore.py diff --git a/modules/inject.py b/src/modules/inject.py similarity index 100% rename from modules/inject.py rename to src/modules/inject.py diff --git a/modules/inline.py b/src/modules/inline.py similarity index 100% rename from modules/inline.py rename to src/modules/inline.py diff --git a/modules/metadata.py b/src/modules/metadata.py similarity index 100% rename from modules/metadata.py rename to src/modules/metadata.py diff --git a/modules/misc.py b/src/modules/misc.py similarity index 100% rename from modules/misc.py rename to src/modules/misc.py diff --git a/modules/patterns.py b/src/modules/patterns.py similarity index 100% rename from modules/patterns.py rename to src/modules/patterns.py diff --git a/modules/requests.py b/src/modules/requests.py similarity index 100% rename from modules/requests.py rename to src/modules/requests.py diff --git a/modules/text.py b/src/modules/text.py similarity index 100% rename from modules/text.py rename to src/modules/text.py diff --git a/modules/userscript.py b/src/modules/userscript.py similarity index 100% rename from modules/userscript.py rename to src/modules/userscript.py diff --git a/modules/utilities.py b/src/modules/utilities.py similarity index 100% rename from modules/utilities.py rename to src/modules/utilities.py From 819584f4bf844cbbf10a20abdcef4e8cc00a637e Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 23 Mar 2020 23:43:10 +0100 Subject: [PATCH 005/137] Readme: Update outdated, Docker-related parts --- README.md | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 14b962e..e71db9e 100644 --- a/README.md +++ b/README.md @@ -36,13 +36,14 @@ Blacklisting or whitelisting is done by giving the `--ignore` or `--intercept` f Examples: -```bash -# Take ignore rules from ignore.txt (included): -python3.6 launcher.py --ignore "ignore.txt" - -# Take intercept rules from all .txt files whose names start with "foo": -python3.6 launcher.py --intercept "foo*.txt" -``` + * Take ignore rules from `rules/ignore.txt` (included): + ```bash + docker run -p 8080:8080 userscript-proxy --ignore "rules/ignore.txt" + ``` + * Take intercept rules from all `.txt` files in the `rules` directory whose names start with `foo`: + ```bash + docker run -p 8080:8080 userscript-proxy --intercept "rules/foo*.txt" + ``` Rules can be specified in two ways: @@ -119,6 +120,14 @@ The [`GM` API][gm-api] and similar runtime facilities are not supported, because ## Options +Options are specified by simply appending them to the `docker run` command, for example: + +```bash +docker run -p 8080:8080 userscript-proxy --transparent +# ^^^^^^^^^^^^ ^^^^^^^^^^^^^ +# args to `docker run` args to Userscript Proxy +``` + ### `--ignore FILE`/`--intercept FILE` Take ignore or intercept rules from `FILE`, which can be a glob pattern matching multiple files. @@ -139,6 +148,18 @@ Insert an HTML comment in each page specifying which userscripts (if any) were i Make mitmproxy listen to TCP port `PORT`. Defaults to `8080`. +**Note:** Be careful when running Userscript Proxy in Docker! If you want to use e.g. port 1337 on the host machine, do this instead: + +```bash +docker run -p 1337:8080 userscript-proxy +``` + +If you really want Userscript Proxy to use a certain port _inside_ the Docker container, e.g. 5555, don't forget to publish that port: + +```bash +docker run -p 1337:5555 userscript-proxy -p 5555 +``` + ### `--query-param-to-disable PARAM`, `-q PARAM` Disable userscripts when the request URL contains `PARAM` as a query parameter. From 7e8152e0ab2cac673b73b6702660754ed8ed4c38 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 25 Mar 2020 22:31:34 +0100 Subject: [PATCH 006/137] Fix typecheck script --- typecheck | 1 + 1 file changed, 1 insertion(+) diff --git a/typecheck b/typecheck index 3651802..a5bfa45 100755 --- a/typecheck +++ b/typecheck @@ -2,6 +2,7 @@ export MYPYPATH=/usr/local/lib/python3.6/dist-packages/ +cd src if [ "$1" == "" ]; then mypy *.py --ignore-missing-imports --follow-imports skip mypy modules/*.py --ignore-missing-imports --follow-imports skip From df4347c3b7defdbe1eb6ba934a638f6c65374184 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Tue, 24 Mar 2020 11:58:24 +0100 Subject: [PATCH 007/137] Use /usr/share/userscripts as default directory --- Dockerfile | 2 +- README.md | 2 +- src/modules/constants.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index aa7fe26..cca1703 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY --from=builder /root/.local /root/.local ENV PATH=/root/.local/bin:$PATH COPY src src COPY rules rules -COPY userscripts userscripts +COPY userscripts /usr/share/userscripts EXPOSE 8080 ENTRYPOINT [ "python", "-u", "src/launcher.py" ] diff --git a/README.md b/README.md index e71db9e..3c99b28 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ In such cases, you have to route traffic from the client to the proxy at the net ### `--userscripts DIR`, `-u DIR` Load userscripts from directory `DIR`. -Defaults to `userscripts`. +Defaults to `/usr/share/userscripts`. [mitmproxy]: https://mitmproxy.org diff --git a/src/modules/constants.py b/src/modules/constants.py index 7c27f11..3ee9e46 100644 --- a/src/modules/constants.py +++ b/src/modules/constants.py @@ -3,5 +3,5 @@ VERSION: str = "0.11.0" ATTRIBUTE_UP_VERSION: str = "data-userscript-proxy-version" DEFAULT_PORT: int = 8080 -DEFAULT_USERSCRIPTS_DIR: str = "userscripts" +DEFAULT_USERSCRIPTS_DIR: str = "/usr/share/userscripts" DEFAULT_QUERY_PARAM_TO_DISABLE: str = "nouserscripts" From a4824c6ec80fee301f50e2669e5d625dccb0388b Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Tue, 24 Mar 2020 15:43:12 +0100 Subject: [PATCH 008/137] Add newline at beginning of script source This is so we get `), never linked (``). @@ -317,6 +312,12 @@ Disable userscripts when the request URL contains `PARAM` as a query parameter. For example, use `-q foo` to disable userscripts for `http://example.com?foo`. Defaults to `nouserscripts`. +### `--rules FILE` + +Take ignore or intercept rules from `FILE`, which can be a glob pattern matching multiple files. +By default, matching traffic is ignored; use `--intercept` to invert this behavior. +See examples above. + ### `--transparent`, `-t` Run mitmproxy in [transparent mode][transparent-mode]. diff --git a/src/launcher.py b/src/launcher.py index 0c6c7d8..f9804a8 100755 --- a/src/launcher.py +++ b/src/launcher.py @@ -35,7 +35,7 @@ def printInfo( if useTransparent: print(f"Please note that ignore/intercept rules based on hostnames may not work in transparent mode; it may be necessary to use IP addresses instead.") else: - print(f"Since {flag(A.no_default_rules)} and neither {flag(A.ignore)} nor {flag(A.intercept)} was given, ALL traffic will be intercepted.") + print(f"Since {flag(A.no_default_rules)} was given and {flag(A.rules)} was not, ALL traffic will be intercepted.") print() @@ -49,18 +49,12 @@ def checkThatUserscriptsDirectoryExistsIfSpecified(directory: str): workingDirectory = os.getcwd() args = getArgparser().parse_args() print(T.WELCOME_MESSAGE) - glob_ignore = args.ignore - glob_intercept = args.intercept - globPattern = ( - glob_intercept if isSomething(glob_intercept) - else glob_ignore if isSomething(glob_ignore) - else None - ) + globPattern = args.rules useCustomFiltering = globPattern is not None useDefaultRules = not args.no_default_rules useTransparent = args.transparent useFiltering = useCustomFiltering or useDefaultRules - useIntercept = isSomething(glob_intercept) + useIntercept = args.intercept is True userscriptsDirectory = args.userscripts_dir checkThatUserscriptsDirectoryExistsIfSpecified(userscriptsDirectory) def ruleFilesContent_default(): diff --git a/src/modules/argparser.py b/src/modules/argparser.py index 3aa839a..2c34f83 100644 --- a/src/modules/argparser.py +++ b/src/modules/argparser.py @@ -6,17 +6,9 @@ def getArgparser(): argparser = ArgumentParser(description=T.description) - eitherIgnoreOrIntercept = argparser.add_mutually_exclusive_group() - eitherIgnoreOrIntercept.add_argument( - flag(A.ignore), - type=str, - metavar=A.metavar_file, - help=A.ignore_help, - ) - eitherIgnoreOrIntercept.add_argument( + argparser.add_argument( flag(A.intercept), - type=str, - metavar=A.metavar_file, + action="store_true", help=A.intercept_help, ) argparser.add_argument( @@ -52,6 +44,12 @@ def getArgparser(): default=A.query_param_to_disable_default, help=A.query_param_to_disable_help, ) + argparser.add_argument( + flag(A.rules), + type=str, + metavar=A.metavar_file, + help=A.rules_help, + ) argparser.add_argument( flag(A.transparent), shortFlag(A.transparent_short), action="store_true", diff --git a/src/modules/arguments.py b/src/modules/arguments.py index 64a690b..7a3fe6e 100644 --- a/src/modules/arguments.py +++ b/src/modules/arguments.py @@ -1,19 +1,18 @@ +from modules.utilities import flag + # Helpers/utilities: metavar_file = "FILE" metavar_dir = "DIR" metavar_param = "PARAM" -matching = f"matching any of the rules specified in {metavar_file} (file name or glob pattern)" - -ignore = "ignore" -ignore_help = "Intercept all traffic except from hosts " + matching +RULES = "rules" inline = "inline" inline_short = "i" inline_help = "Always insert userscripts inline, never linked" intercept = "intercept" -intercept_help = "Intercept only traffic from hosts " + matching +intercept_help = f"Invert the meaning of {flag(RULES)} so that traffic from matched hosts is intercepted instead of ignored" list_injected = "list-injected" list_injected_short = "l" @@ -35,6 +34,10 @@ query_param_to_disable_default = "nouserscripts" query_param_to_disable_help = f"""Disable userscripts when the request URL contains a PARAM query parameter, for example "foo" to disable userscripts for http://example.com?foo (default: {query_param_to_disable_default})""" +rules = RULES +rules_short = "r" +rules_help = f"Ignore (or, with {flag(intercept)}, intercept) traffic from hosts matching any of the rules specified in {metavar_file} (file name or glob pattern)" + transparent = "transparent" transparent_short = "t" transparent_help = "Transparent mode" From 783b59124f5ec109522ccf8bb63a271744e8836f Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 12 Apr 2020 19:02:58 +0200 Subject: [PATCH 044/137] Readme: Add Contribute section --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index f2dc80c..389a33c 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,18 @@ In such cases, you have to route traffic from the client to the proxy at the net Load userscripts from directory `DIR`. +## Contribute + +How to build and run from source: + +``` +git clone https://github.com/SimonAlling/userscript-proxy +cd userscript-proxy +docker build -t userscript-proxy . +docker run --rm --name userscript-proxy -p 8080:8080 userscript-proxy +``` + + [mitmproxy]: https://mitmproxy.org [minification]: https://en.wikipedia.org/wiki/Minification_(programming) [metadata]: https://wiki.greasespot.net/Metadata_Block From 4769c491ee09880b277b3f256cc245338b26beb0 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 12 Apr 2020 19:46:59 +0200 Subject: [PATCH 045/137] Add Makefile --- Makefile | 2 ++ README.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..89602cf --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +image: + docker build -t userscript-proxy . diff --git a/README.md b/README.md index 389a33c..699cfad 100644 --- a/README.md +++ b/README.md @@ -338,7 +338,7 @@ How to build and run from source: ``` git clone https://github.com/SimonAlling/userscript-proxy cd userscript-proxy -docker build -t userscript-proxy . +make image docker run --rm --name userscript-proxy -p 8080:8080 userscript-proxy ``` From 4fca2fe67f847f17fb4d5928d77c01418f13a6d2 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 12 Apr 2020 19:55:17 +0200 Subject: [PATCH 046/137] Decrease heading levels The primary reason is so that we can generate a nice TOC later. --- README.md | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 699cfad..d411cc5 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ Userscript Proxy is built around [mitmproxy][mitmproxy] and acts as a MITM, inje Both HTTP and HTTPS are supported. -## How to use it +# How to use it -### Security notice +## Security notice Make sure you understand these security aspects before using Userscript Proxy: @@ -17,7 +17,7 @@ Make sure you understand these security aspects before using Userscript Proxy: * **You should not expose the proxy to the Internet** (i.e. outside your LAN), because then anyone can connect to it and use your Internet connection for whatever they want. In practice, this means that you should _not_ add a port-forward for Userscript Proxy in your router. -### Getting started +## Getting started 1. Make sure you have [Docker](https://www.docker.com) installed. This should work: @@ -67,7 +67,7 @@ Make sure you understand these security aspects before using Userscript Proxy: 1. Visit [`http://example.com`](http://example.com). You should see a green page and a message saying that Userscript Proxy is working. -### On a mobile device +## On a mobile device 1. You need to know the local IP address of the machine running Userscript Proxy (i.e. where you ran `docker run` above). This is usually something like `192.168.1.67`. @@ -88,7 +88,7 @@ Make sure you understand these security aspects before using Userscript Proxy: 1. Visit [`http://example.com`](http://example.com) on your mobile device. You should see the same green page as above. -### HTTPS +## HTTPS When you've set up Userscript Proxy on your mobile device as described above, you'll notice that you can't visit sites via HTTPS anymore. This is because your device thinks you're being [MITM'd](https://en.wikipedia.org/wiki/Man-in-the-middle_attack) (which, technically, you are – but that's exactly what we want). @@ -121,12 +121,12 @@ Otherwise, read on. 1. Install the certificate. - #### Android + ### Android Follow the on-screen instructions. If you're asked to choose between **VPN and apps** and **Wi-Fi**, choose **VPN and apps**. - #### iOS + ### iOS 1. Tap **Allow** when asked if you want to download a configuration profile. @@ -142,7 +142,7 @@ Otherwise, read on. 1. You should now be able to browse via HTTPS as usual. -### Deploying userscripts +## Deploying userscripts Userscript Proxy comes with one single userscript, useful only for testing that the proxy is up and running. To use userscripts you've downloaded or written yourself, you need to tell Userscript Proxy where they are. @@ -157,7 +157,7 @@ To use userscripts you've downloaded or written yourself, you need to tell Users ``` -## Apps with certificate pinning +# Apps with certificate pinning Apps like App Store and Facebook Messenger refuse to connect through a MITM proxy, so their traffic must be ignored by mitmproxy. There are two approaches: @@ -185,7 +185,7 @@ Examples: Rules can be specified in two ways: -### Basic pattern +## Basic pattern Based on the syntax used by userscript `@include` directives. Asterisk (`*`) means any string (including the empty string). @@ -194,7 +194,7 @@ Asterisk (`*`) means any string (including the empty string). To match a domain without matching all of its subdomains, use a regex rule instead (see below). -#### Examples +### Examples | Rule | Matches | |----------------|-----------------------------------------------------------------| @@ -203,7 +203,7 @@ To match a domain without matching all of its subdomains, use a regex rule inste | `*cdn.net` | `cdn.net`, `fbcdn.net` and `x.fbcdn.net`, but not `cdn.net.com` | | `site.com:80` | `site.com:80` and `x.site.com:80`, but not `site.com:443` | -### Regular expression +## Regular expression If a rule starts and ends with a slash (`/`), it is treated as a Python regex. @@ -215,7 +215,7 @@ Also, be careful with `$`: A regex like `/site.com$/` will never match, because Anything from a `#` until the end of the line is treated as a comment. Leading and trailing whitespace have no effect. -#### Examples +### Examples | Rule | Matches | |-----------------|-------------------------------------------------------------------| @@ -223,7 +223,7 @@ Leading and trailing whitespace have no effect. | `/^site\.com:/` | `site.com`, but not `x.site.com`, `mysite.com` or `site.com.net` | -## Data usage +# Data usage Userscript Proxy has no data usage impact when no userscript is injected, i.e. for URLs without any matching userscript. When a script _is_ injected, **exactly one** of the following things happens: @@ -238,7 +238,7 @@ Userscripts are injected into _every_ response from a matching URL, and the size If the `@downloadURL` approach is not possible, for one reason or the other, it is a good idea to be aware of this issue, and to take appropriate action such as [minifying][minification] userscripts and adding suitable ignore rules. -## Userscript compatibility +# Userscript compatibility Userscript Proxy supports (a subset of) the [Greasemonkey metadata syntax][metadata]. No adaptation of plain JavaScript userscripts should be required. @@ -256,7 +256,7 @@ These directives are supported: The [`GM` API][gm-api] and similar runtime facilities are not supported, because userscripts can only be injected as regular scripts. -## Options +# Options Options are specified by simply appending them to the `docker run` command, for example: @@ -266,24 +266,24 @@ docker run --rm --name userscript-proxy -p 8080:8080 userscript-proxy --transpar # flags to `docker run` flags to Userscript Proxy ``` -### `--inline`, `-i` +## `--inline`, `-i` Always inject scripts inline (``), never linked (``). Useful to test new userscript features without having to re-upload the userscript and clear browser cache. -### `--list-injected`, `-l` +## `--list-injected`, `-l` Insert an HTML comment in each page specifying which userscripts (if any) were injected. -### `--no-default-rules` +## `--no-default-rules` Skip built-in default rules, which are otherwise automatically applied so that common apps like App Store and Facebook Messenger work out of the box. -### `--no-default-userscripts` +## `--no-default-userscripts` Skip loading built-in default userscripts intended for sanity checks and similar purposes, e.g. Example Userscript. -### `--port PORT`, `-p PORT` +## `--port PORT`, `-p PORT` Make mitmproxy listen to TCP port `PORT`. Defaults to `8080`. @@ -306,19 +306,19 @@ Or you can let the Docker container be a part of the host's network: docker run --rm --network host --name userscript-proxy userscript-proxy -p 5555 ``` -### `--query-param-to-disable PARAM`, `-q PARAM` +## `--query-param-to-disable PARAM`, `-q PARAM` Disable userscripts when the request URL contains `PARAM` as a query parameter. For example, use `-q foo` to disable userscripts for `http://example.com?foo`. Defaults to `nouserscripts`. -### `--rules FILE` +## `--rules FILE` Take ignore or intercept rules from `FILE`, which can be a glob pattern matching multiple files. By default, matching traffic is ignored; use `--intercept` to invert this behavior. See examples above. -### `--transparent`, `-t` +## `--transparent`, `-t` Run mitmproxy in [transparent mode][transparent-mode]. Useful if you cannot set a proxy in the client, e.g. when using OpenVPN Connect on Android to connect to a VPN server on the network where your proxy is running. @@ -326,12 +326,12 @@ In such cases, you have to route traffic from the client to the proxy at the net **NOTE:** In transparent mode, ignore/intercept rules based on hostname (rather than IP address) may not work, because mitmproxy may not be able to see the hostname of responses without intercepting them. -### `--userscripts-dir DIR`, `-u DIR` +## `--userscripts-dir DIR`, `-u DIR` Load userscripts from directory `DIR`. -## Contribute +# Contribute How to build and run from source: From c42d3d1e60b73a860f313651a12611c201bcac53 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 12 Apr 2020 20:01:00 +0200 Subject: [PATCH 047/137] Improve 'Getting started' headings --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d411cc5..8dfed2f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Userscript Proxy is built around [mitmproxy][mitmproxy] and acts as a MITM, inje Both HTTP and HTTPS are supported. -# How to use it +# Getting started ## Security notice @@ -17,7 +17,7 @@ Make sure you understand these security aspects before using Userscript Proxy: * **You should not expose the proxy to the Internet** (i.e. outside your LAN), because then anyone can connect to it and use your Internet connection for whatever they want. In practice, this means that you should _not_ add a port-forward for Userscript Proxy in your router. -## Getting started +## Starting the proxy 1. Make sure you have [Docker](https://www.docker.com) installed. This should work: From 0202b917acdddd82d14f0e706e26a0b07c5fc6af Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 12 Apr 2020 19:57:03 +0200 Subject: [PATCH 048/137] Add readme TOC and 'docs' Make target --- .gitignore | 3 +++ Makefile | 15 +++++++++++++++ README.md | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/.gitignore b/.gitignore index 7bbc71c..870917d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Project-specific stuff: +gh-md-toc + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Makefile b/Makefile index 89602cf..daaf736 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,17 @@ +TOC_FILE = gh-md-toc +TOC_HASH = 042fc595336c3a39f82b1edbafdf2afd2503d9930d192fcfda757aa65522c14c +TOC_URL = https://raw.githubusercontent.com/ekalinin/github-markdown-toc/56f7c5939e2119bed86291ddba9fb6c2ee61fb09/gh-md-toc + +docs: + wget -O $(TOC_FILE) $(TOC_URL) + # Check that the file hasn't been tampered with: + echo "$(TOC_HASH) $(TOC_FILE)" | sha256sum -c + chmod +x $(TOC_FILE) + # Generate and insert TOC: + ./$(TOC_FILE) --insert README.md + # Remove files created by gh-md-toc: + rm README.md.orig.* + rm README.md.toc.* + image: docker build -t userscript-proxy . diff --git a/README.md b/README.md index 8dfed2f..a517738 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,39 @@ No jailbreak/root required. Userscript Proxy is built around [mitmproxy][mitmproxy] and acts as a MITM, injecting matching userscripts into web pages as they flow through it. Both HTTP and HTTPS are supported. + + * [Userscript Proxy](#userscript-proxy) + * [Getting started](#getting-started) + * [Security notice](#security-notice) + * [Starting the proxy](#starting-the-proxy) + * [On a mobile device](#on-a-mobile-device) + * [HTTPS](#https) + * [Android](#android) + * [iOS](#ios) + * [Deploying userscripts](#deploying-userscripts) + * [Apps with certificate pinning](#apps-with-certificate-pinning) + * [Basic pattern](#basic-pattern) + * [Examples](#examples) + * [Regular expression](#regular-expression) + * [Examples](#examples-1) + * [Data usage](#data-usage) + * [Userscript compatibility](#userscript-compatibility) + * [Options](#options) + * [--inline, -i](#--inline--i) + * [--list-injected, -l](#--list-injected--l) + * [--no-default-rules](#--no-default-rules) + * [--no-default-userscripts](#--no-default-userscripts) + * [--port PORT, -p PORT](#--port-port--p-port) + * [--query-param-to-disable PARAM, -q PARAM](#--query-param-to-disable-param--q-param) + * [--rules FILE](#--rules-file) + * [--transparent, -t](#--transparent--t) + * [--userscripts-dir DIR, -u DIR](#--userscripts-dir-dir--u-dir) + * [Contribute](#contribute) + + + + + # Getting started From 881881851eec01566f81f8412f3442b226bb1681 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 12 Apr 2020 20:12:42 +0200 Subject: [PATCH 049/137] Add 'release' Make target --- Makefile | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index daaf736..1e0c77f 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,13 @@ TOC_FILE = gh-md-toc TOC_HASH = 042fc595336c3a39f82b1edbafdf2afd2503d9930d192fcfda757aa65522c14c TOC_URL = https://raw.githubusercontent.com/ekalinin/github-markdown-toc/56f7c5939e2119bed86291ddba9fb6c2ee61fb09/gh-md-toc +TAG ?= latest + +FILE_WITH_VERSION = src/modules/constants.py +DOCKER_USER = alling +DOCKER_REPO = userscript-proxy +DOCKER_FULL = $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) + docs: wget -O $(TOC_FILE) $(TOC_URL) # Check that the file hasn't been tampered with: @@ -14,4 +21,14 @@ docs: rm README.md.toc.* image: - docker build -t userscript-proxy . + docker build -t $(DOCKER_FULL) . + +release: image +ifneq "$(shell git status --porcelain)" "" + $(error Working directory not clean) +endif + # Update in-app version: + sed -i 's/^VERSION: str = "[^"]*"/VERSION: str = "$(TAG)"/' $(FILE_WITH_VERSION) + git add $(FILE_WITH_VERSION) + git commit -m "v$(TAG)" + git tag "v$(TAG)" From 31b0eee9af726067108829f2572847437d1b4f67 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 12 Apr 2020 22:53:53 +0200 Subject: [PATCH 050/137] Add 'start' Make target --- Makefile | 7 +++++++ README.md | 3 +-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1e0c77f..d1c5bfe 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,10 @@ FILE_WITH_VERSION = src/modules/constants.py DOCKER_USER = alling DOCKER_REPO = userscript-proxy DOCKER_FULL = $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) +# Can be anything: +CA_VOLUME = mitmproxy-ca +# Needs to match where mitmproxy stores its CA: +CA_DIR = /root/.mitmproxy docs: wget -O $(TOC_FILE) $(TOC_URL) @@ -32,3 +36,6 @@ endif git add $(FILE_WITH_VERSION) git commit -m "v$(TAG)" git tag "v$(TAG)" + +start: image + docker run --rm -p 8080:8080 --name $(DOCKER_REPO) -v "$(CA_VOLUME):$(CA_DIR)" $(DOCKER_FULL) diff --git a/README.md b/README.md index a517738..92a4792 100644 --- a/README.md +++ b/README.md @@ -371,8 +371,7 @@ How to build and run from source: ``` git clone https://github.com/SimonAlling/userscript-proxy cd userscript-proxy -make image -docker run --rm --name userscript-proxy -p 8080:8080 userscript-proxy +make start ``` From d0a54e6076fdf08fe7ea41a0aab376926184c9c1 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 16:07:43 +0200 Subject: [PATCH 051/137] Make 'image' the default Make target --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index d1c5bfe..a6f85c1 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,8 @@ CA_VOLUME = mitmproxy-ca # Needs to match where mitmproxy stores its CA: CA_DIR = /root/.mitmproxy +.DEFAULT_GOAL := image + docs: wget -O $(TOC_FILE) $(TOC_URL) # Check that the file hasn't been tampered with: From bcfa9af82975ed1c5160aa3546890a97627b94cb Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 16:08:00 +0200 Subject: [PATCH 052/137] Check tag in 'release' Make target --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a6f85c1..b6c3587 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,8 @@ TOC_FILE = gh-md-toc TOC_HASH = 042fc595336c3a39f82b1edbafdf2afd2503d9930d192fcfda757aa65522c14c TOC_URL = https://raw.githubusercontent.com/ekalinin/github-markdown-toc/56f7c5939e2119bed86291ddba9fb6c2ee61fb09/gh-md-toc -TAG ?= latest +DEFAULT_TAG = latest +TAG ?= $(DEFAULT_TAG) FILE_WITH_VERSION = src/modules/constants.py DOCKER_USER = alling @@ -32,6 +33,9 @@ image: release: image ifneq "$(shell git status --porcelain)" "" $(error Working directory not clean) +endif +ifeq "$(TAG)" "$(DEFAULT_TAG)" + $(error Please specify a version (e.g. TAG="1.2.3")) endif # Update in-app version: sed -i 's/^VERSION: str = "[^"]*"/VERSION: str = "$(TAG)"/' $(FILE_WITH_VERSION) From d9da1d139e1686ee48fc53e787011bc374e4f7f5 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 16:10:10 +0200 Subject: [PATCH 053/137] Improve loaded userscripts info formatting --- src/injector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/injector.py b/src/injector.py index 91bf92c..9641d02 100644 --- a/src/injector.py +++ b/src/injector.py @@ -119,6 +119,7 @@ def loadUserscripts(directory: str) -> List[Userscript]: os.chdir(workingDirectory) # so mitmproxy does not unload the script logInfo("") logInfo(str(len(loadedUserscripts)) + " userscript(s) loaded:") + logInfo("") logInfo(bulletList(map( lambda s: f"{first(s).name} ({second(s)})", loadedUserscripts From d6cd8d04318f155fe028d244d8c26c603b097ac6 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 18:08:30 +0200 Subject: [PATCH 054/137] Fix colored output in Docker --- Makefile | 3 ++- README.md | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index b6c3587..e6949fc 100644 --- a/Makefile +++ b/Makefile @@ -44,4 +44,5 @@ endif git tag "v$(TAG)" start: image - docker run --rm -p 8080:8080 --name $(DOCKER_REPO) -v "$(CA_VOLUME):$(CA_DIR)" $(DOCKER_FULL) + docker run -t --rm -p 8080:8080 --name $(DOCKER_REPO) -v "$(CA_VOLUME):$(CA_DIR)" $(DOCKER_FULL) +# The -t flag enables colored output. diff --git a/README.md b/README.md index 92a4792..0609856 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ Make sure you understand these security aspects before using Userscript Proxy: 1. Start Userscript Proxy: ``` - docker run --rm --name userscript-proxy -p 8080:8080 userscript-proxy + docker run -t --rm --name userscript-proxy -p 8080:8080 userscript-proxy ``` When you see _Proxy server listening at http://*:8080_, the proxy is up and running. @@ -136,7 +136,7 @@ Otherwise, read on. Then start it again, this time with the `-v` flag as shown below: ``` - docker run --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" userscript-proxy + docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" userscript-proxy ``` This creates a new Docker volume and mounts it at `/root/.mitmproxy`, where mitmproxy stores its certificate authority files. @@ -186,7 +186,7 @@ To use userscripts you've downloaded or written yourself, you need to tell Users 1. Run Userscript Proxy with your userscripts directory mounted at some location (e.g. `/userscripts`) inside the Docker container, and tell Userscript Proxy to read userscripts from that directory: ``` - docker run --rm --name userscript-proxy -p 8080:8080 -v "/home/alling/userscripts:/userscripts" userscript-proxy --userscripts-dir "/userscripts" + docker run -t --rm --name userscript-proxy -p 8080:8080 -v "/home/alling/userscripts:/userscripts" userscript-proxy --userscripts-dir "/userscripts" ``` @@ -209,11 +209,11 @@ Examples: * Take ignore rules from `/home/alling/rules/ignore.txt`: ```bash - docker run --rm -v "/home/alling/rules:/rules" userscript-proxy --rules "/rules/ignore.txt" + docker run -t --rm -v "/home/alling/rules:/rules" userscript-proxy --rules "/rules/ignore.txt" ``` * Take intercept rules from all `.txt` files in the `/home/alling/rules` directory whose names start with `foo`: ```bash - docker run --rm -v "/home/alling/rules:/rules" userscript-proxy --rules "/rules/foo*.txt" --intercept + docker run -t --rm -v "/home/alling/rules:/rules" userscript-proxy --rules "/rules/foo*.txt" --intercept ``` Rules can be specified in two ways: @@ -294,9 +294,9 @@ The [`GM` API][gm-api] and similar runtime facilities are not supported, because Options are specified by simply appending them to the `docker run` command, for example: ```bash -docker run --rm --name userscript-proxy -p 8080:8080 userscript-proxy --transparent -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ -# flags to `docker run` flags to Userscript Proxy +docker run -t --rm --name userscript-proxy -p 8080:8080 userscript-proxy --transparent +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ +# flags to `docker run` flags to Userscript Proxy ``` ## `--inline`, `-i` @@ -324,19 +324,19 @@ Defaults to `8080`. **Note:** Be careful when running Userscript Proxy in Docker! If you want to use e.g. port 1337 on the host machine, do this instead: ```bash -docker run --rm --name userscript-proxy -p 1337:8080 userscript-proxy +docker run -t --rm --name userscript-proxy -p 1337:8080 userscript-proxy ``` If you really want Userscript Proxy to use a certain port _inside_ the Docker container, e.g. 5555, don't forget to publish that port: ```bash -docker run --rm --name userscript-proxy -p 1337:5555 userscript-proxy -p 5555 +docker run -t --rm --name userscript-proxy -p 1337:5555 userscript-proxy -p 5555 ``` Or you can let the Docker container be a part of the host's network: ```bash -docker run --rm --network host --name userscript-proxy userscript-proxy -p 5555 +docker run -t --rm --network host --name userscript-proxy userscript-proxy -p 5555 ``` ## `--query-param-to-disable PARAM`, `-q PARAM` From 6321ad9a673a82c2c1fe3e1aabb1e1de6c20a17d Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 18:13:46 +0200 Subject: [PATCH 055/137] Warn about inline injection when no @downloadURL --- src/injector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/injector.py b/src/injector.py index 9641d02..b44ca61 100644 --- a/src/injector.py +++ b/src/injector.py @@ -105,6 +105,8 @@ def loadUserscripts(directory: str) -> List[Userscript]: continue try: script = userscript.create(content) + if script.downloadURL is None: + logWarning(f"""{script.name} will be injected inline because it does not have a {metadata.PREFIX_TAG}{userscript.directive_downloadURL}.""") loadedUserscripts.append((script, filename)) if script.downloadURL is None and len(script.unsafeSequences) > 0: logError(unsafeSequencesMessage(script)) From f5b4482f42d13242bcc17197c38461b341c70e6b Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 18:41:53 +0200 Subject: [PATCH 056/137] Add and use metadata.tag function --- src/injector.py | 4 ++-- src/modules/metadata.py | 21 ++++++++++++--------- src/modules/userscript.py | 6 +++--- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/injector.py b/src/injector.py index b44ca61..f3fc8cb 100644 --- a/src/injector.py +++ b/src/injector.py @@ -73,7 +73,7 @@ def unsafeSequencesMessage(script: Userscript) -> str: Possible solutions: """ + bulletList([ f"Make sure the userscript does not contain any of the sequences listed above.", - f"Make the userscript available online and give it a {metadata.PREFIX_TAG}{userscript.directive_downloadURL}", + f"Make the userscript available online and give it a {metadata.tag(userscript.directive_downloadURL)}", f"Remove the {flag(A.inline)} flag.", ]) @@ -106,7 +106,7 @@ def loadUserscripts(directory: str) -> List[Userscript]: try: script = userscript.create(content) if script.downloadURL is None: - logWarning(f"""{script.name} will be injected inline because it does not have a {metadata.PREFIX_TAG}{userscript.directive_downloadURL}.""") + logWarning(f"""{script.name} will be injected inline because it does not have a {metadata.tag(userscript.directive_downloadURL)}.""") loadedUserscripts.append((script, filename)) if script.downloadURL is None and len(script.unsafeSequences) > 0: logError(unsafeSequencesMessage(script)) diff --git a/src/modules/metadata.py b/src/modules/metadata.py index 5936a51..1787bfb 100644 --- a/src/modules/metadata.py +++ b/src/modules/metadata.py @@ -39,6 +39,9 @@ class Tag_boolean(NamedTuple): BLOCK_START: str = "==UserScript==" BLOCK_END: str = "==/UserScript==" +def tag(name: str) -> str: + return PREFIX_TAG + name + REGEXGROUP_CONTENT: str = "content" REGEX_METADATA_BLOCK: Pattern = re.compile( PREFIX_COMMENT + r"\s*" + BLOCK_START + r"\n" @@ -58,19 +61,19 @@ class Tag_boolean(NamedTuple): STRING_ERROR_MISSING_BLOCK: str = f"""No metadata block found. The metadata block must follow this format: {PREFIX_COMMENT} {BLOCK_START} - {PREFIX_COMMENT} {PREFIX_TAG}key1 value1 - {PREFIX_COMMENT} {PREFIX_TAG}key2 value2 + {PREFIX_COMMENT} {tag("key1")} value1 + {PREFIX_COMMENT} {tag("key2")} value2 {PREFIX_COMMENT} ... - {PREFIX_COMMENT} {PREFIX_TAG}keyN valueN + {PREFIX_COMMENT} {tag("keyN")} valueN {PREFIX_COMMENT} {BLOCK_END} -It must start with `{PREFIX_COMMENT} {BLOCK_START}` and end with `{PREFIX_COMMENT} {BLOCK_END}`, and every line must be a line comment starting with an {PREFIX_TAG}-prefixed tag name, then whitespace, then a tag value (with the exception of boolean directives such as {PREFIX_TAG}noframes, which are automatically true if present). +It must start with `{PREFIX_COMMENT} {BLOCK_START}` and end with `{PREFIX_COMMENT} {BLOCK_END}`, and every line must be a line comment starting with an {PREFIX_TAG}-prefixed tag name, then whitespace, then a tag value (with the exception of boolean directives such as {tag("noframes")}, which are automatically true if present). """ STRING_ERROR_INVALID_BLOCK: Template = Template(f"""Invalid metadata block. Only comments are allowed, and each line should follow this format: - {PREFIX_COMMENT} {PREFIX_TAG}key value + {PREFIX_COMMENT} {tag("key")} value This line does not: @@ -78,15 +81,15 @@ class Tag_boolean(NamedTuple): """) -STRING_ERROR_MISSING_TAG: Template = Template(f"""The {PREFIX_TAG}$tagName metadata directive is required, but was not found.""") +STRING_ERROR_MISSING_TAG: Template = Template(f"""The {tag("$tagName")} metadata directive is required, but was not found.""") -STRING_ERROR_MISSING_VALUE: Template = Template(f"""The {PREFIX_TAG}$tagName metadata directive requires a value, like so: +STRING_ERROR_MISSING_VALUE: Template = Template(f"""The {tag("$tagName")} metadata directive requires a value, like so: - {PREFIX_COMMENT} {PREFIX_TAG}$tagName something + {PREFIX_COMMENT} {tag("$tagName")} something """) -STRING_ERROR_PREDICATE_FAILED: Template = Template(f"""Detected a {PREFIX_TAG}$tagName metadata directive with an invalid value, namely: +STRING_ERROR_PREDICATE_FAILED: Template = Template(f"""Detected a {tag("$tagName")} metadata directive with an invalid value, namely: $tagValue diff --git a/src/modules/userscript.py b/src/modules/userscript.py index b874dd8..00cf58a 100644 --- a/src/modules/userscript.py +++ b/src/modules/userscript.py @@ -7,7 +7,7 @@ import modules.inline as inline import modules.metadata as metadata -from modules.metadata import PREFIX_TAG, Metadata, Tag, Tag_boolean, Tag_string +from modules.metadata import Metadata, Tag, Tag_boolean, Tag_string from modules.patterns import isIncludePattern, isMatchPattern, regexFromIncludePattern from modules.utilities import compose2, isSomething, stripIndentation, strs @@ -101,7 +101,7 @@ def __init__(self,*args,**kwargs): validateMetadata: Callable[[Metadata], Metadata] = metadata.validator(METADATA_TAGS) -STRING_WARNING_INVALID_REGEX: Template = Template(f"""{PREFIX_TAG}{directive_include}/{PREFIX_TAG}{directive_exclude} patterns starting and ending with `/` are interpreted as regular expressions, and this pattern is not a valid regex: +STRING_WARNING_INVALID_REGEX: Template = Template(f"""{metadata.tag(directive_include)}/{metadata.tag(directive_exclude)} patterns starting and ending with `/` are interpreted as regular expressions, and this pattern is not a valid regex: $pattern @@ -181,7 +181,7 @@ def withEventListener(event: str) -> Callable[[str], str]: def withNoframes(scriptContent: str) -> str: return stripIndentation(f""" - if (window.top === window) {{ // {PREFIX_TAG + directive_noframes} + if (window.top === window) {{ // {metadata.tag(directive_noframes)} {scriptContent} }} """) From 32b41b40a27b62c94852a2372628d881724ba025 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 18:49:45 +0200 Subject: [PATCH 057/137] Add 'all' Make target and make it the default --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e6949fc..05507bf 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,8 @@ CA_VOLUME = mitmproxy-ca # Needs to match where mitmproxy stores its CA: CA_DIR = /root/.mitmproxy -.DEFAULT_GOAL := image +.PHONY : all +all: image docs: wget -O $(TOC_FILE) $(TOC_URL) From cdeae6253a1c0e06992f6328cf87634aa2519946 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 18:55:49 +0200 Subject: [PATCH 058/137] Add 'install' Make target --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 05507bf..a53e2f3 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,9 @@ docs: image: docker build -t $(DOCKER_FULL) . +install: image + docker image inspect $(DOCKER_FULL) > /dev/null + release: image ifneq "$(shell git status --porcelain)" "" $(error Working directory not clean) From 68c7ee180326c3ffa7b485ff42b53c30cba324c3 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 19:06:39 +0200 Subject: [PATCH 059/137] Fix Makefile comments --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index a53e2f3..a33cf8b 100644 --- a/Makefile +++ b/Makefile @@ -19,12 +19,12 @@ all: image docs: wget -O $(TOC_FILE) $(TOC_URL) - # Check that the file hasn't been tampered with: +# Check that the file hasn't been tampered with: echo "$(TOC_HASH) $(TOC_FILE)" | sha256sum -c chmod +x $(TOC_FILE) - # Generate and insert TOC: +# Generate and insert TOC: ./$(TOC_FILE) --insert README.md - # Remove files created by gh-md-toc: +# Remove files created by gh-md-toc: rm README.md.orig.* rm README.md.toc.* @@ -41,12 +41,12 @@ endif ifeq "$(TAG)" "$(DEFAULT_TAG)" $(error Please specify a version (e.g. TAG="1.2.3")) endif - # Update in-app version: +# Update in-app version: sed -i 's/^VERSION: str = "[^"]*"/VERSION: str = "$(TAG)"/' $(FILE_WITH_VERSION) git add $(FILE_WITH_VERSION) git commit -m "v$(TAG)" git tag "v$(TAG)" start: image +# The -t flag enables colored output: docker run -t --rm -p 8080:8080 --name $(DOCKER_REPO) -v "$(CA_VOLUME):$(CA_DIR)" $(DOCKER_FULL) -# The -t flag enables colored output. From 5d36ac56c63008a41f76629228dbcabc16880323 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 16:04:51 +0200 Subject: [PATCH 060/137] Readme: Run from Docker Hub instead of locally --- README.md | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 0609856..768e1d9 100644 --- a/README.md +++ b/README.md @@ -59,25 +59,10 @@ Make sure you understand these security aspects before using Userscript Proxy: docker --version ``` -1. If you have [Git](https://git-scm.com) installed, clone the repo: - - ``` - git clone https://github.com/SimonAlling/userscript-proxy - cd userscript-proxy - ``` - - Otherwise, you can [download the code as a ZIP file](https://github.com/SimonAlling/userscript-proxy/archive/master.zip), extract it and `cd` into the extracted `userscript-proxy` directory. - -1. Build: - - ``` - docker build -t userscript-proxy . - ``` - 1. Start Userscript Proxy: ``` - docker run -t --rm --name userscript-proxy -p 8080:8080 userscript-proxy + docker run -t --rm --name userscript-proxy -p 8080:8080 alling/userscript-proxy ``` When you see _Proxy server listening at http://*:8080_, the proxy is up and running. @@ -136,7 +121,7 @@ Otherwise, read on. Then start it again, this time with the `-v` flag as shown below: ``` - docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" userscript-proxy + docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" alling/userscript-proxy ``` This creates a new Docker volume and mounts it at `/root/.mitmproxy`, where mitmproxy stores its certificate authority files. @@ -186,7 +171,7 @@ To use userscripts you've downloaded or written yourself, you need to tell Users 1. Run Userscript Proxy with your userscripts directory mounted at some location (e.g. `/userscripts`) inside the Docker container, and tell Userscript Proxy to read userscripts from that directory: ``` - docker run -t --rm --name userscript-proxy -p 8080:8080 -v "/home/alling/userscripts:/userscripts" userscript-proxy --userscripts-dir "/userscripts" + docker run -t --rm --name userscript-proxy -p 8080:8080 -v "/home/alling/userscripts:/userscripts" alling/userscript-proxy --userscripts-dir "/userscripts" ``` @@ -209,11 +194,11 @@ Examples: * Take ignore rules from `/home/alling/rules/ignore.txt`: ```bash - docker run -t --rm -v "/home/alling/rules:/rules" userscript-proxy --rules "/rules/ignore.txt" + docker run -t --rm -v "/home/alling/rules:/rules" alling/userscript-proxy --rules "/rules/ignore.txt" ``` * Take intercept rules from all `.txt` files in the `/home/alling/rules` directory whose names start with `foo`: ```bash - docker run -t --rm -v "/home/alling/rules:/rules" userscript-proxy --rules "/rules/foo*.txt" --intercept + docker run -t --rm -v "/home/alling/rules:/rules" alling/userscript-proxy --rules "/rules/foo*.txt" --intercept ``` Rules can be specified in two ways: @@ -294,9 +279,9 @@ The [`GM` API][gm-api] and similar runtime facilities are not supported, because Options are specified by simply appending them to the `docker run` command, for example: ```bash -docker run -t --rm --name userscript-proxy -p 8080:8080 userscript-proxy --transparent -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ -# flags to `docker run` flags to Userscript Proxy +docker run -t --rm --name userscript-proxy -p 8080:8080 alling/userscript-proxy --transparent +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ +# flags to `docker run` flags to Userscript Proxy ``` ## `--inline`, `-i` @@ -324,19 +309,19 @@ Defaults to `8080`. **Note:** Be careful when running Userscript Proxy in Docker! If you want to use e.g. port 1337 on the host machine, do this instead: ```bash -docker run -t --rm --name userscript-proxy -p 1337:8080 userscript-proxy +docker run -t --rm --name userscript-proxy -p 1337:8080 alling/userscript-proxy ``` If you really want Userscript Proxy to use a certain port _inside_ the Docker container, e.g. 5555, don't forget to publish that port: ```bash -docker run -t --rm --name userscript-proxy -p 1337:5555 userscript-proxy -p 5555 +docker run -t --rm --name userscript-proxy -p 1337:5555 alling/userscript-proxy -p 5555 ``` Or you can let the Docker container be a part of the host's network: ```bash -docker run -t --rm --network host --name userscript-proxy userscript-proxy -p 5555 +docker run -t --rm --network host --name userscript-proxy alling/userscript-proxy -p 5555 ``` ## `--query-param-to-disable PARAM`, `-q PARAM` From 93db6d7440817b16e2a228371c89694bbc3a4a8a Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 21:24:22 +0200 Subject: [PATCH 061/137] Remove unused executables in Docker image --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aa025e9..cf99265 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,8 @@ FROM base WORKDIR /app -COPY --from=builder /root/.local /root/.local +COPY --from=builder /root/.local/lib /root/.local/lib +COPY --from=builder /root/.local/bin/mitmdump /root/.local/bin/mitmdump ENV PATH=/root/.local/bin:$PATH COPY src src COPY default-rules default-rules From a841bfeb2370bd4851634fb8cf54f0df140db0ad Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 21:36:40 +0200 Subject: [PATCH 062/137] Fix 'release' Make target The release image now contains the VERSION change. --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a33cf8b..f20d7f5 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ image: install: image docker image inspect $(DOCKER_FULL) > /dev/null -release: image +release: ifneq "$(shell git status --porcelain)" "" $(error Working directory not clean) endif @@ -43,6 +43,7 @@ ifeq "$(TAG)" "$(DEFAULT_TAG)" endif # Update in-app version: sed -i 's/^VERSION: str = "[^"]*"/VERSION: str = "$(TAG)"/' $(FILE_WITH_VERSION) + docker build -t $(DOCKER_FULL) . git add $(FILE_WITH_VERSION) git commit -m "v$(TAG)" git tag "v$(TAG)" From 99388f6253ece4221b89aa55d2dec308c65ff3d5 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 21:38:24 +0200 Subject: [PATCH 063/137] v1.0.0 --- src/modules/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/constants.py b/src/modules/constants.py index 0ec3e1c..186e765 100644 --- a/src/modules/constants.py +++ b/src/modules/constants.py @@ -1,6 +1,6 @@ VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" -VERSION: str = "0.11.0" +VERSION: str = "1.0.0" ATTRIBUTE_UP_VERSION: str = "data-userscript-proxy-version" DEFAULT_RULES_DIR: str = "default-rules/" From fb964bab101249786ae13aa44a388c437e045d98 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 21:56:48 +0200 Subject: [PATCH 064/137] Tag with latest when making release --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f20d7f5..7f55c74 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ TAG ?= $(DEFAULT_TAG) FILE_WITH_VERSION = src/modules/constants.py DOCKER_USER = alling DOCKER_REPO = userscript-proxy -DOCKER_FULL = $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) # Can be anything: CA_VOLUME = mitmproxy-ca # Needs to match where mitmproxy stores its CA: @@ -29,10 +28,10 @@ docs: rm README.md.toc.* image: - docker build -t $(DOCKER_FULL) . + docker build -t $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) . install: image - docker image inspect $(DOCKER_FULL) > /dev/null + docker image inspect $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) > /dev/null release: ifneq "$(shell git status --porcelain)" "" @@ -43,11 +42,12 @@ ifeq "$(TAG)" "$(DEFAULT_TAG)" endif # Update in-app version: sed -i 's/^VERSION: str = "[^"]*"/VERSION: str = "$(TAG)"/' $(FILE_WITH_VERSION) - docker build -t $(DOCKER_FULL) . + docker build -t $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) . + docker tag $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) $(DOCKER_USER)/$(DOCKER_REPO):$(DEFAULT_TAG) git add $(FILE_WITH_VERSION) git commit -m "v$(TAG)" git tag "v$(TAG)" start: image # The -t flag enables colored output: - docker run -t --rm -p 8080:8080 --name $(DOCKER_REPO) -v "$(CA_VOLUME):$(CA_DIR)" $(DOCKER_FULL) + docker run -t --rm -p 8080:8080 --name $(DOCKER_REPO) -v "$(CA_VOLUME):$(CA_DIR)" $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) From 36e99e3140b2bc8551bc96c334aaea8517ae1bf4 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Apr 2020 22:07:24 +0200 Subject: [PATCH 065/137] Update license copyright --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 3770507..9811b04 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017 Simon Alling +Copyright (c) 2017–2020 Simon Alling Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 89699de0c1973df0fcfb8837665b92f276fa9101 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 26 Apr 2020 22:20:46 +0200 Subject: [PATCH 066/137] Readme: Clarify intro summary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 768e1d9..f013bcc 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Browser extensions on iOS, Android and pretty much any other web browsing device. No jailbreak/root required. -Userscript Proxy is built around [mitmproxy][mitmproxy] and acts as a MITM, injecting matching userscripts into web pages as they flow through it. +Userscript Proxy is built around [mitmproxy][mitmproxy] and acts as a MITM, injecting userscripts into web pages as they flow through it. Both HTTP and HTTPS are supported. From 616db844f2126a8f6667126b327b9ae05ccf5c12 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 26 Apr 2020 22:21:26 +0200 Subject: [PATCH 067/137] Readme: Clarify first security aspect --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f013bcc..7ed0591 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Both HTTP and HTTPS are supported. Make sure you understand these security aspects before using Userscript Proxy: - * **You should run the proxy on your own server**, because it can read and modify all HTTP(S) traffic sent through it. + * **You should run the proxy on your own server**, because it can read and modify all traffic sent through it. * **You should not expose the proxy to the Internet** (i.e. outside your LAN), because then anyone can connect to it and use your Internet connection for whatever they want. In practice, this means that you should _not_ add a port-forward for Userscript Proxy in your router. From 631ea8ae59e96de624e60b98dc5cbf29f298fccc Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 26 Apr 2020 22:27:54 +0200 Subject: [PATCH 068/137] Readme: Clarify second security aspect --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ed0591..de3f181 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,9 @@ Both HTTP and HTTPS are supported. Make sure you understand these security aspects before using Userscript Proxy: * **You should run the proxy on your own server**, because it can read and modify all traffic sent through it. - * **You should not expose the proxy to the Internet** (i.e. outside your LAN), because then anyone can connect to it and use your Internet connection for whatever they want. + * **You should not expose the proxy to incoming connections from the Internet**, because then anyone can connect to it and use your Internet connection for whatever they want. In practice, this means that you should _not_ add a port-forward for Userscript Proxy in your router. + (Browsing the web via the proxy uses only outgoing connections, which is fine.) ## Starting the proxy From df03f57e950b912befb273b1d448c50dc108e3cd Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 26 Apr 2020 22:29:30 +0200 Subject: [PATCH 069/137] Readme: Fix incorrect indentation in list --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index de3f181..2ce6164 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ Make sure you understand these security aspects before using Userscript Proxy: curl --proxy localhost:8080 http://example.com ``` - The output should contain a ``), never linked (``). diff --git a/src/injector.py b/src/injector.py index f3fc8cb..8db983c 100644 --- a/src/injector.py +++ b/src/injector.py @@ -9,6 +9,7 @@ import modules.arguments as A import modules.constants as C +import modules.csp as csp import modules.inject as inject import modules.inline as inline import modules.metadata as metadata @@ -139,6 +140,7 @@ def load(self, loader): loader.add_option(sanitize(A.inline), bool, False, A.inline_help) loader.add_option(sanitize(A.no_default_userscripts), bool, False, A.no_default_userscripts_help) loader.add_option(sanitize(A.list_injected), bool, False, A.list_injected_help) + loader.add_option(sanitize(A.bypass_csp), Optional[str], A.bypass_csp_default, A.bypass_csp_help) loader.add_option(sanitize(A.userscripts_dir), Optional[str], A.userscripts_dir_default, A.userscripts_dir_help) loader.add_option(sanitize(A.query_param_to_disable), str, A.query_param_to_disable_default, A.query_param_to_disable_help) @@ -167,7 +169,7 @@ def response(self, flow: http.HTTPFlow): if CONTENT_TYPE in response.headers: if any(map(lambda t: t in response.headers[CONTENT_TYPE], RELEVANT_CONTENT_TYPES)): # Response is a web page; proceed. - insertedScripts: List[str] = [] + injections: List[csp.Injection] = [] soup = BeautifulSoup( response.content, HTML_PARSER, @@ -185,23 +187,31 @@ def response(self, flow: http.HTTPFlow): logError(unsafeSequencesMessage(script)) continue logInfo(f"""Injecting {script.name}{"" if script.version is None else " " + C.VERSION_PREFIX + script.version} into {requestURL} ({"inline" if useInline else "linked"}) ...""") + shouldUseNonce = useInline and option(A.bypass_csp) == A.bypass_csp_script # If not inline, then URL is used for bypassing; if bypass for nothing or everything, then the nonce would have no effect anyway. + nonce = csp.generateNonce() if shouldUseNonce else None result = inject.inject(script, soup, inject.Options( inline = option(A.inline), + nonce = nonce )) if type(result) is BeautifulSoup: soup = result - insertedScripts.append(script.name + ("" if script.version is None else " " + T.stringifyVersion(script.version))) + injections.append(csp.Injection( + userscript = script, + nonce = nonce, + )) else: logError("Injection failed due to the following error:") logError(str(result)) + handleContentSecurityPolicy(response, injections) index_DTD: Optional[int] = indexOfDTD(soup) # Insert information comment: if option(A.list_injected): + namesOfInjectedScripts = [ i.userscript.name + ("" if i.userscript.version is None else " " + T.stringifyVersion(i.userscript.version)) for i in injections ] soup.insert(0 if index_DTD is None else 1+index_DTD, Comment( HTML_INFO_COMMENT_PREFIX + ( - "No matching userscripts for this URL." if insertedScripts == [] - else "These scripts were inserted:\n" + bulletList(insertedScripts) + "No matching userscripts for this URL." if namesOfInjectedScripts == [] + else "These scripts were inserted:\n" + bulletList(namesOfInjectedScripts) ) + "\n" )) # Serialize and encode: @@ -211,4 +221,19 @@ def response(self, flow: http.HTTPFlow): ) +def handleContentSecurityPolicy(response: http.HTTPFlow.response, injections: List[csp.Injection]): + # If there is a CSP header, we may need to modify it for the userscript(s) to work. + ContentSecurityPolicy = "Content-Security-Policy" + if ContentSecurityPolicy in response.headers: + bypassCspValue = option(A.bypass_csp) + if bypassCspValue == A.bypass_csp_script: + logInfo(f"Bypassing host site's Content Security Policy for userscripts only (not any resources injected _by_ userscripts, such as stylesheets and images). Try `{flag(A.bypass_csp)} {A.bypass_csp_everything}` if something does not work properly.") + response.headers[ContentSecurityPolicy] = csp.headerWithScriptsAllowed(response.headers[ContentSecurityPolicy], injections) + elif bypassCspValue == A.bypass_csp_everything: + logInfo(f"Bypassing host site's Content Security Policy altogether due to `{flag(A.bypass_csp)} {A.bypass_csp_everything}`.") + del response.headers[ContentSecurityPolicy] + else: + logWarning(f"Host site has a Content Security Policy. Try the {flag(A.bypass_csp)} flag if userscripts don't work properly.") + + addons = [ UserscriptInjector() ] diff --git a/src/launcher.py b/src/launcher.py index f9804a8..f868cb8 100755 --- a/src/launcher.py +++ b/src/launcher.py @@ -55,6 +55,7 @@ def checkThatUserscriptsDirectoryExistsIfSpecified(directory: str): useTransparent = args.transparent useFiltering = useCustomFiltering or useDefaultRules useIntercept = args.intercept is True + bypassCsp = args.bypass_csp userscriptsDirectory = args.userscripts_dir checkThatUserscriptsDirectoryExistsIfSpecified(userscriptsDirectory) def ruleFilesContent_default(): @@ -101,6 +102,7 @@ def ruleFilesContent_custom(): "--set", f"""{sanitize(A.inline)}={str(args.inline).lower()}""", "--set", f"""{sanitize(A.list_injected)}={str(args.list_injected).lower()}""", "--set", f"""{sanitize(A.no_default_userscripts)}={str(args.no_default_userscripts).lower()}""", + "--set", "" if bypassCsp is None else f"""{sanitize(A.bypass_csp)}={bypassCsp}""", "--set", "" if userscriptsDirectory is None else f"""{sanitize(A.userscripts_dir)}={userscriptsDirectory}""", "--set", f"""{sanitize(A.query_param_to_disable)}={args.query_param_to_disable}""", # Empty string breaks the argument chain: diff --git a/src/modules/argparser.py b/src/modules/argparser.py index 2c34f83..4a7f525 100644 --- a/src/modules/argparser.py +++ b/src/modules/argparser.py @@ -6,6 +6,14 @@ def getArgparser(): argparser = ArgumentParser(description=T.description) + argparser.add_argument( + flag(A.bypass_csp), + type=str, + metavar=A.metavar_allow, + choices=A.bypass_csp_values, + default=A.bypass_csp_default, + help=A.bypass_csp_help, + ) argparser.add_argument( flag(A.intercept), action="store_true", diff --git a/src/modules/arguments.py b/src/modules/arguments.py index 7a3fe6e..022c2ac 100644 --- a/src/modules/arguments.py +++ b/src/modules/arguments.py @@ -4,9 +4,18 @@ metavar_file = "FILE" metavar_dir = "DIR" metavar_param = "PARAM" +metavar_allow = "ALLOW" RULES = "rules" +bypass_csp = "bypass-csp" +bypass_csp_nothing = "nothing" +bypass_csp_script = "script" +bypass_csp_everything = "everything" +bypass_csp_default = bypass_csp_script +bypass_csp_values = { bypass_csp_nothing, bypass_csp_script, bypass_csp_everything } +bypass_csp_help = f"Bypass host site's Content Security Policy to allow userscripts to run properly. If {metavar_allow} is '{bypass_csp_script}', the CSP is bypassed only for the userscript itself. Use '{bypass_csp_everything}' to allow everything, which may be necessary if the userscript injects CSS, images etc. Note that the latter completely disables any CSP from every host site into which a userscript is injected. Default: '{bypass_csp_default}'." + inline = "inline" inline_short = "i" inline_help = "Always insert userscripts inline, never linked" diff --git a/src/modules/csp.py b/src/modules/csp.py new file mode 100644 index 0000000..9a6f41f --- /dev/null +++ b/src/modules/csp.py @@ -0,0 +1,40 @@ +import secrets +from typing import List, NamedTuple, Optional + +from modules.userscript import Userscript +from modules.utilities import isSomething + +# Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy + + +class Injection(NamedTuple): + userscript: Userscript + nonce: Optional[str] + + +def headerWithScriptsAllowed(cspHeaderValue: str, injections: List[Injection]) -> str: + # Example CSP header: + # + # Content-Security-Policy: default-src 'self'; frame-src 'self'; img-src https:; connect-src 'self' + # + cspKeyValuePairs = [ directive.strip().split(" ", 1) for directive in cspHeaderValue.split(';') ] + cspDict = { key: value for key, value in cspKeyValuePairs } + if "script-src" not in cspDict: + # Browsers fall back to default-src if there is no script-src. + # Since there was no script-src directive and we are adding one, we include the default-src (if present) in it to avoid breaking the site's effective CSP. + cspDict["script-src"] = cspDict["default-src"] if "default-src" in cspDict else "" + sourcesToAllow = [ source(i) for i in injections ] + cspDict["script-src"] += " " + " ".join(sourcesToAllow) + return '; '.join([ f'{key} {value}' for key, value in cspDict.items() ]) + + +def source(injection: Injection) -> str: + if isSomething(injection.nonce): + return f"'nonce-{injection.nonce}'" + else: + # MDN about host (i.e. download URL) sources: "Unlike other values below, single quotes shouldn't be used." + return injection.userscript.downloadURL + + +def generateNonce(): + return secrets.token_hex() # If no argument is passed, "a reasonable default is used" for the number of bytes. diff --git a/src/modules/inject.py b/src/modules/inject.py index b6de896..69a6f5f 100644 --- a/src/modules/inject.py +++ b/src/modules/inject.py @@ -1,19 +1,22 @@ -from typing import NamedTuple, Union +from typing import NamedTuple, Optional, Union from bs4 import BeautifulSoup, Tag import modules.constants as C import modules.userscript as userscript from modules.userscript import Userscript, document_end, document_idle -from modules.utilities import fromOptional, idem, stripIndentation +from modules.utilities import fromOptional, idem, isSomething, stripIndentation class Options(NamedTuple): inline: bool + nonce: Optional[str] def inject(script: Userscript, soup: BeautifulSoup, options: Options) -> Union[BeautifulSoup, Exception]: useInline = options.inline or script.downloadURL is None tag = soup.new_tag("script") + if isSomething(options.nonce): + tag["nonce"] = options.nonce # Used to bypass CSP for inline-injected userscripts. tag[C.ATTRIBUTE_UP_VERSION] = C.VERSION withLoadListenerIfRunAtIdle = userscript.withEventListener("load") if script.runAt == document_idle else idem withNoframesIfNoframes = userscript.withNoframes if script.noframes else idem From 048067f5bda9a7f2e4efc5b96de84052ff1fa7c0 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Tue, 23 Mar 2021 20:35:21 +0100 Subject: [PATCH 078/137] v1.1.0 (#9) * Add feature to bypass Content Security Policy (#7) --- src/modules/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/constants.py b/src/modules/constants.py index 186e765..264c6a4 100644 --- a/src/modules/constants.py +++ b/src/modules/constants.py @@ -1,6 +1,6 @@ VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" -VERSION: str = "1.0.0" +VERSION: str = "1.1.0" ATTRIBUTE_UP_VERSION: str = "data-userscript-proxy-version" DEFAULT_RULES_DIR: str = "default-rules/" From 2900deec63aa02e6e0a649c36441b8f8dc761be4 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 21:14:07 +0200 Subject: [PATCH 079/137] Pin dependency versions Now when I returned to this project a few years later, of course it doesn't work anymore. `make start` successfully builds a Docker image, but it fails like this after printing the list of hosts that will be ignored: Traceback (most recent call last): File "/root/.local/bin/mitmdump", line 8, in sys.exit(mitmdump()) File "/root/.local/lib/python3.7/site-packages/mitmproxy/tools/_main.py", line 153, in mitmdump from mitmproxy.tools import dump File "/root/.local/lib/python3.7/site-packages/mitmproxy/tools/dump.py", line 1, in from mitmproxy import addons File "/root/.local/lib/python3.7/site-packages/mitmproxy/addons/__init__.py", line 12, in from mitmproxy.addons import onboarding File "/root/.local/lib/python3.7/site-packages/mitmproxy/addons/onboarding.py", line 2, in from mitmproxy.addons.onboardingapp import app File "/root/.local/lib/python3.7/site-packages/mitmproxy/addons/onboardingapp/__init__.py", line 3, in from flask import Flask, render_template File "/root/.local/lib/python3.7/site-packages/flask/__init__.py", line 14, in from jinja2 import escape File "/root/.local/lib/python3.7/site-packages/jinja2/__init__.py", line 12, in from .environment import Environment File "/root/.local/lib/python3.7/site-packages/jinja2/environment.py", line 25, in from .defaults import BLOCK_END_STRING File "/root/.local/lib/python3.7/site-packages/jinja2/defaults.py", line 3, in from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 File "/root/.local/lib/python3.7/site-packages/jinja2/filters.py", line 13, in from markupsafe import soft_unicode ImportError: cannot import name 'soft_unicode' from 'markupsafe' (/root/.local/lib/python3.7/site-packages/markupsafe/__init__.py) I asked ChatGPT what this was about and got this explanation: > The error you're seeing is due to a **breaking change in the `markupsafe` package**, specifically: > > * As of `markupsafe >= 2.1.0`, the `soft_unicode` function has been **removed**. > > * Older packages like `Jinja2` (or any other dependency that hasn't been updated to match) **still try to import** `soft_unicode`. And yeah, [indeed](https://github.com/pallets/markupsafe/issues/282). ChatGPT proposed three different solutions: * Pin `markupsafe` to a compatible version like `<2.1.0`. * Upgrade `Jinja2`, `Flask`, or any other package that uses `markupsafe`. * Freeze a working `requirements.txt` from 2021. I chose to go for the latter, which I acheived like this: ```bash docker run -it --rm --name userscript-proxy -p 8080:8080 --entrypoint /bin/bash alling/userscript-proxy:1.1.0 # Inside the container: python --version # Printed "Python 3.7.10" pip freeze > /tmp/requirements.txt # Ctrl + D to exit, then: docker cp userscript-proxy:/tmp/requirements.txt . ``` Also, from now on, we won't be hard-wrapping commit messages at 72 characters anymore. --- Dockerfile | 2 +- requirements.txt | 44 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf99265..d3367ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7-slim AS base +FROM python:3.7.10-slim AS base FROM base AS builder diff --git a/requirements.txt b/requirements.txt index 196f515..f73c0f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,40 @@ -mitmproxy -beautifulsoup4 -urlmatch -lxml +asgiref==3.3.1 +beautifulsoup4==4.9.3 +blinker==1.4 +Brotli==1.0.9 +certifi==2020.12.5 +cffi==1.14.5 +click==7.1.2 +cryptography==3.2.1 +Flask==1.1.2 +h11==0.12.0 +h2==4.0.0 +hpack==4.0.0 +hyperframe==6.0.0 +itsdangerous==1.1.0 +Jinja2==2.11.3 +kaitaistruct==0.9 +ldap3==2.8.1 +lxml==4.6.3 +MarkupSafe==1.1.1 +mitmproxy==5.3.0 +msgpack==1.0.2 +passlib==1.7.4 +protobuf==3.13.0 +publicsuffix2==2.20191221 +pyasn1==0.4.8 +pycparser==2.20 +pyOpenSSL==19.1.0 +pyparsing==2.4.7 +pyperclip==1.8.2 +ruamel.yaml==0.16.13 +ruamel.yaml.clib==0.2.2 +six==1.15.0 +sortedcontainers==2.2.2 +soupsieve==2.2.1 +tornado==6.1 +urlmatch==1.0.1 +urwid==2.1.2 +Werkzeug==1.0.1 +wsproto==0.15.0 +zstandard==0.14.1 From 68f51c7a42bd8fe0659a878c1538976e46e16b35 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 23:01:09 +0200 Subject: [PATCH 080/137] Remove readme TOC and 'docs' Make target This commit essentially reverts 0202b917acdddd82d14f0e706e26a0b07c5fc6af. Maintaining the table of contents involves manual steps and s error-prone. For example, now that I ran `make docs`, it just erased the entire TOC in `README.md`. Given that GitHub [generates] a TOC automatically nowadays, removing our homegrown setup is an easy choice. [generates]: https://github.blog/changelog/2021-04-13-table-of-contents-support-in-markdown-files/ --- .gitignore | 3 --- Makefile | 15 --------------- README.md | 34 ---------------------------------- 3 files changed, 52 deletions(-) diff --git a/.gitignore b/.gitignore index 870917d..7bbc71c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -# Project-specific stuff: -gh-md-toc - # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Makefile b/Makefile index 7f55c74..bb75a06 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,3 @@ -TOC_FILE = gh-md-toc -TOC_HASH = 042fc595336c3a39f82b1edbafdf2afd2503d9930d192fcfda757aa65522c14c -TOC_URL = https://raw.githubusercontent.com/ekalinin/github-markdown-toc/56f7c5939e2119bed86291ddba9fb6c2ee61fb09/gh-md-toc - DEFAULT_TAG = latest TAG ?= $(DEFAULT_TAG) @@ -16,17 +12,6 @@ CA_DIR = /root/.mitmproxy .PHONY : all all: image -docs: - wget -O $(TOC_FILE) $(TOC_URL) -# Check that the file hasn't been tampered with: - echo "$(TOC_HASH) $(TOC_FILE)" | sha256sum -c - chmod +x $(TOC_FILE) -# Generate and insert TOC: - ./$(TOC_FILE) --insert README.md -# Remove files created by gh-md-toc: - rm README.md.orig.* - rm README.md.toc.* - image: docker build -t $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) . diff --git a/README.md b/README.md index 5ce5ef3..fff7eed 100644 --- a/README.md +++ b/README.md @@ -6,40 +6,6 @@ No jailbreak/root required. Userscript Proxy is built around [mitmproxy][mitmproxy] and acts as a MITM, injecting userscripts into web pages as they flow through it. Both HTTP and HTTPS are supported. - - * [Userscript Proxy](#userscript-proxy) - * [Getting started](#getting-started) - * [Security notice](#security-notice) - * [Starting the proxy](#starting-the-proxy) - * [On a mobile device](#on-a-mobile-device) - * [HTTPS](#https) - * [Android](#android) - * [iOS](#ios) - * [Deploying userscripts](#deploying-userscripts) - * [Apps with certificate pinning](#apps-with-certificate-pinning) - * [Basic pattern](#basic-pattern) - * [Examples](#examples) - * [Regular expression](#regular-expression) - * [Examples](#examples-1) - * [Data usage](#data-usage) - * [Userscript compatibility](#userscript-compatibility) - * [Options](#options) - * [--bypass-csp ALLOW](#--bypass-csp-allow) - * [--inline, -i](#--inline--i) - * [--list-injected, -l](#--list-injected--l) - * [--no-default-rules](#--no-default-rules) - * [--no-default-userscripts](#--no-default-userscripts) - * [--port PORT, -p PORT](#--port-port--p-port) - * [--query-param-to-disable PARAM, -q PARAM](#--query-param-to-disable-param--q-param) - * [--rules FILE](#--rules-file) - * [--transparent, -t](#--transparent--t) - * [--userscripts-dir DIR, -u DIR](#--userscripts-dir-dir--u-dir) - * [Contribute](#contribute) - - - - - # Getting started From e92411675fb7ae10d49f0962096d38255dafaa6f Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 23:23:41 +0200 Subject: [PATCH 081/137] Readme: Make documentation a bit less verbose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💡 `git show --color-words='This should .+|\w+|.'` --- README.md | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index fff7eed..bb0b04e 100644 --- a/README.md +++ b/README.md @@ -21,11 +21,6 @@ Make sure you understand these security aspects before using Userscript Proxy: ## Starting the proxy 1. Make sure you have [Docker](https://www.docker.com) installed. - This should work: - - ``` - docker --version - ``` 1. Start Userscript Proxy: @@ -59,14 +54,12 @@ Make sure you understand these security aspects before using Userscript Proxy: This is usually something like `192.168.1.67`. You can typically [find it](https://google.com/search?q=find+local+IP+address) by running `ip a`, `ifconfig` or `ipconfig` depending on your operating system. - If your local IP address is `192.168.1.67`, and the proxy is running (see above), this should work: + This should work on any computer within the LAN: ``` curl --proxy 192.168.1.67:8080 http://example.com ``` -1. Your mobile device needs to be on the same LAN as your proxy, so make sure it's connected to your Wi-Fi. - 1. On your mobile device, go to the settings for the currently active Wi-Fi connection. Find the proxy settings, select **Manual proxy** or similar, and set `192.168.1.67` with port `8080`. @@ -130,7 +123,7 @@ Otherwise, read on. ## Deploying userscripts Userscript Proxy comes with one single userscript, useful only for testing that the proxy is up and running. -To use userscripts you've downloaded or written yourself, you need to tell Userscript Proxy where they are. +To use userscripts you've downloaded or written yourself: 1. You need the **absolute path** to a directory containing your userscripts. This could be something like `/home/alling/userscripts`. @@ -223,7 +216,7 @@ A userscript is injected by reference if and only if it has a specified `@downlo (This can be overridden using the `--inline` flag, in which case all userscripts are injected inline.) Userscripts are injected into _every_ response from a matching URL, and the size of a userscript can be anything from a few hundred bytes for the most basic ones to hundreds of kilobytes in extreme cases, so there are _massive_ data usage reductions to be gained from making the userscript accessible by URL and including a `@downloadURL`. -If the `@downloadURL` approach is not possible, for one reason or the other, it is a good idea to be aware of this issue, and to take appropriate action such as [minifying][minification] userscripts and adding suitable ignore rules. +If the `@downloadURL` approach is not possible, for one reason or the other, it is a good idea to be aware of this issue, and to take appropriate action such as [minifying] userscripts and adding suitable ignore rules. # Userscript compatibility @@ -340,7 +333,7 @@ make start [mitmproxy]: https://mitmproxy.org -[minification]: https://en.wikipedia.org/wiki/Minification_(programming) +[minifying]: https://en.wikipedia.org/wiki/Minification_(programming) [metadata]: https://wiki.greasespot.net/Metadata_Block [transparent-mode]: https://docs.mitmproxy.org/stable/concepts-modes/#transparent-proxy [gm-api]: https://wiki.greasespot.net/GM.getValue From fe0827984315c9cc6f6724257affbcdf32955549 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 23:26:47 +0200 Subject: [PATCH 082/137] Readme: Clarify custom userscripts/rules documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two explanatory bullet points are removed because I feel like the addition of `my-` makes them superfluous. 💡 `git show --color-words=.` --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bb0b04e..0d880d5 100644 --- a/README.md +++ b/README.md @@ -131,12 +131,9 @@ To use userscripts you've downloaded or written yourself: 1. Run Userscript Proxy like this: ``` - docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" -v "/home/alling/userscripts:/userscripts" alling/userscript-proxy --userscripts-dir "/userscripts" + docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" -v "/home/alling/userscripts:/my-userscripts" alling/userscript-proxy --userscripts-dir "/my-userscripts" ``` - * `-v "/home/alling/userscripts:/userscripts"` mounts your userscripts directory at `/userscripts` inside the Docker container. - * `--userscripts-dir "/userscripts"` tells Userscript Proxy to read userscripts from `/userscripts`. - # Apps with certificate pinning @@ -157,11 +154,11 @@ Examples: * Take ignore rules from `/home/alling/rules/ignore.txt`: ```bash - docker run -t --rm -v "/home/alling/rules:/rules" alling/userscript-proxy --rules "/rules/ignore.txt" + docker run -t --rm -v "/home/alling/rules:/my-rules" alling/userscript-proxy --rules "/my-rules/ignore.txt" ``` * Take intercept rules from all `.txt` files in the `/home/alling/rules` directory whose names start with `foo`: ```bash - docker run -t --rm -v "/home/alling/rules:/rules" alling/userscript-proxy --rules "/rules/foo*.txt" --intercept + docker run -t --rm -v "/home/alling/rules:/my-rules" alling/userscript-proxy --rules "/my-rules/foo*.txt" --intercept ``` Rules can be specified in two ways: From 18801ca7a736ae70300f59ef8b3f50ea086dbd17 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 23:29:33 +0200 Subject: [PATCH 083/137] Readme: Simplify HTTPS documentation Most likely, users always want HTTPS support, so why act like it's something optional that they _might_ want to add later? This commit makes the very first suggested command launch Userscript Proxy with persistent certificates, and also removes some extraneous details about HTTPS and certificates that users probably aren't interested in. --- README.md | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 0d880d5..e3d54df 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Make sure you understand these security aspects before using Userscript Proxy: 1. Start Userscript Proxy: ``` - docker run -t --rm --name userscript-proxy -p 8080:8080 alling/userscript-proxy + docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" alling/userscript-proxy ``` When you see _Proxy server listening at http://*:8080_, the proxy is up and running. @@ -68,29 +68,7 @@ Make sure you understand these security aspects before using Userscript Proxy: ## HTTPS -When you've set up Userscript Proxy on your mobile device as described above, you'll notice that you can't visit sites via HTTPS anymore. -This is because your device thinks you're being [MITM'd](https://en.wikipedia.org/wiki/Man-in-the-middle_attack) (which, technically, you are – by yourself). - -To make HTTPS connections work, you need to tell your device that it should trust your proxy. -This is accomplished by installing a certificate. - -**In general, installing a certificate might pose a security risk. If you don't trust me and mitmproxy, stop here.** -Otherwise, read on. - -1. Stop the proxy by pressing `Ctrl` + `C` in the terminal where it's running. - Then start it again, this time with the `-v` flag as shown below: - - ``` - docker run -t --rm --name userscript-proxy -p 8080:8080 -v "mitmproxy-ca:/root/.mitmproxy" alling/userscript-proxy - ``` - - This creates a new Docker volume and mounts it at `/root/.mitmproxy`, where mitmproxy stores its certificate authority files. - This is necessary so that you can restart the proxy later without having to perform all these steps again. - - In this example, `mitmproxy-ca` is the name of the new Docker volume. - You can choose any name you want, as long as it's not already in use. - -1. Make sure your mobile device is configured to use the proxy as decribed above. +To make HTTPS work, you need to make your device trust your proxy by installing a certificate generated by mitmproxy. 1. On your mobile device, go to [http://mitm.it](http://mitm.it). You should see icons for Apple, Windows, Android, etc. @@ -118,8 +96,6 @@ Otherwise, read on. 1. Under _Enable full trust for root certificates_, enable **mitmproxy**, confirming the action if prompted. -1. You should now be able to browse via HTTPS as usual. - ## Deploying userscripts Userscript Proxy comes with one single userscript, useful only for testing that the proxy is up and running. From e0e9aa3f7a9f847fec7d8b60f0af9549dbffca6a Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 28 May 2025 23:39:57 +0200 Subject: [PATCH 084/137] Readme: Improve local-IP-address documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit About the changes in this commit: 1. It doesn't help the user to know that the command they should use depends on their OS. They'll figure it out anyway. 2. I feel like the `curl` command belongs in some kind of troubleshooting section, rather than the happy path. Also, it can give the impression that Userscript Proxy will always be running at 192.168.1.67. 3. Adding "e.g." should hopefully make it more obvious that 192.168.1.67 is just an example. 💡 `git show --color-words='curl.+com|.'` --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e3d54df..18353ac 100644 --- a/README.md +++ b/README.md @@ -52,16 +52,10 @@ Make sure you understand these security aspects before using Userscript Proxy: 1. You need to know the local IP address of the machine running Userscript Proxy (i.e. where you ran `docker run` above). This is usually something like `192.168.1.67`. - You can typically [find it](https://google.com/search?q=find+local+IP+address) by running `ip a`, `ifconfig` or `ipconfig` depending on your operating system. - - This should work on any computer within the LAN: - - ``` - curl --proxy 192.168.1.67:8080 http://example.com - ``` + You can typically [find it](https://google.com/search?q=find+local+IP+address) by running `ip a`, `ifconfig` or `ipconfig`. 1. On your mobile device, go to the settings for the currently active Wi-Fi connection. - Find the proxy settings, select **Manual proxy** or similar, and set `192.168.1.67` with port `8080`. + Find the proxy settings, select **Manual proxy** or similar, and set e.g. `192.168.1.67` with port `8080`. 1. Visit [`http://example.com`](http://example.com) on your mobile device. You should see the same green page as above. From ef6c883445042af49a940b3396ae22c775076fed Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 29 May 2025 00:20:10 +0200 Subject: [PATCH 085/137] Readme: Add Docker Compose file --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 18353ac..85e9caa 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,32 @@ Both HTTP and HTTPS are supported. # Getting started +If you're familiar with Userscript Proxy, you might want to use Docker Compose: + +```yaml +services: + userscript-proxy: + image: alling/userscript-proxy:v1.1.0 + container_name: userscript-proxy + command: + - --userscripts-dir + - /my-userscripts + - --rules + - /my-rules/ignore.txt + ports: + - "8080:8080" + volumes: + - mitmproxy-ca:/root/.mitmproxy + - /absolute/path/to/my/userscripts/:/my-userscripts # Modify the part before the ':'! + - /absolute/path/to/my/rules/:/my-rules # Modify the part before the ':'! + restart: always + +volumes: + mitmproxy-ca: +``` + +Otherwise, keep reading. + ## Security notice Make sure you understand these security aspects before using Userscript Proxy: From b485dbbff6b78ab61b79e01d7f0478a0830d842a Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 29 May 2025 00:32:11 +0200 Subject: [PATCH 086/137] Readme: Fix incorrect image tag in Compose file --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 85e9caa..38401ba 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ If you're familiar with Userscript Proxy, you might want to use Docker Compose: ```yaml services: userscript-proxy: - image: alling/userscript-proxy:v1.1.0 + image: alling/userscript-proxy:1.1.0 container_name: userscript-proxy command: - --userscripts-dir From 87a2a3cf07642dea8612af71409a1d170fe8589a Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 29 May 2025 11:58:17 +0200 Subject: [PATCH 087/137] Typecheck code with mypy in Dockerfile --- Dockerfile | 4 +++- requirements.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d3367ac..120681b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,17 +6,19 @@ COPY requirements.txt . # We're not going to run anything in the build container, so we'll suppress the script location warnings. RUN pip install --user --no-warn-script-location -r requirements.txt - FROM base WORKDIR /app COPY --from=builder /root/.local/lib /root/.local/lib COPY --from=builder /root/.local/bin/mitmdump /root/.local/bin/mitmdump +COPY --from=builder /root/.local/bin/mypy /root/.local/bin/mypy ENV PATH=/root/.local/bin:$PATH +COPY typecheck . COPY src src COPY default-rules default-rules COPY default-userscripts default-userscripts +RUN ./typecheck EXPOSE 8080 ENTRYPOINT [ "python", "-u", "src/launcher.py" ] diff --git a/requirements.txt b/requirements.txt index f73c0f8..cf24d23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ lxml==4.6.3 MarkupSafe==1.1.1 mitmproxy==5.3.0 msgpack==1.0.2 +mypy==1.4.1 passlib==1.7.4 protobuf==3.13.0 publicsuffix2==2.20191221 From 56f5a432a619ed018d27ebefc1bad89ee3541782 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 29 May 2025 12:01:15 +0200 Subject: [PATCH 088/137] Remove unused `isSomething` import It became unused in 4c55b27e299cb33d7895346de9f21bf07e3041ff. --- src/launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/launcher.py b/src/launcher.py index f868cb8..7eebc9b 100755 --- a/src/launcher.py +++ b/src/launcher.py @@ -12,7 +12,7 @@ import modules.ignore as ignore from modules.misc import sanitize import modules.text as T -from modules.utilities import flag, idem, isSomething, itemList +from modules.utilities import flag, idem, itemList FILENAME_INJECTOR: str = "injector.py" MATCH_NO_HOSTS = r"^$" From 3416bb510f51fe0779a9e9478d6cc7362c0b075e Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 29 May 2025 12:42:14 +0200 Subject: [PATCH 089/137] Remove `isSomething` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I feel like it just adds an unnecessary layer of indirection. 💡 `git show --color-words='\w+|.'` --- src/modules/csp.py | 3 +-- src/modules/inject.py | 4 ++-- src/modules/metadata.py | 6 +++--- src/modules/patterns.py | 8 ++++---- src/modules/userscript.py | 12 ++++++------ src/modules/utilities.py | 4 ---- 6 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/modules/csp.py b/src/modules/csp.py index 9a6f41f..230eb98 100644 --- a/src/modules/csp.py +++ b/src/modules/csp.py @@ -2,7 +2,6 @@ from typing import List, NamedTuple, Optional from modules.userscript import Userscript -from modules.utilities import isSomething # Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy @@ -29,7 +28,7 @@ def headerWithScriptsAllowed(cspHeaderValue: str, injections: List[Injection]) - def source(injection: Injection) -> str: - if isSomething(injection.nonce): + if injection.nonce is not None: return f"'nonce-{injection.nonce}'" else: # MDN about host (i.e. download URL) sources: "Unlike other values below, single quotes shouldn't be used." diff --git a/src/modules/inject.py b/src/modules/inject.py index 69a6f5f..4f639c1 100644 --- a/src/modules/inject.py +++ b/src/modules/inject.py @@ -5,7 +5,7 @@ import modules.constants as C import modules.userscript as userscript from modules.userscript import Userscript, document_end, document_idle -from modules.utilities import fromOptional, idem, isSomething, stripIndentation +from modules.utilities import fromOptional, idem, stripIndentation class Options(NamedTuple): inline: bool @@ -15,7 +15,7 @@ class Options(NamedTuple): def inject(script: Userscript, soup: BeautifulSoup, options: Options) -> Union[BeautifulSoup, Exception]: useInline = options.inline or script.downloadURL is None tag = soup.new_tag("script") - if isSomething(options.nonce): + if options.nonce is not None: tag["nonce"] = options.nonce # Used to bypass CSP for inline-injected userscripts. tag[C.ATTRIBUTE_UP_VERSION] = C.VERSION withLoadListenerIfRunAtIdle = userscript.withEventListener("load") if script.runAt == document_idle else idem diff --git a/src/modules/metadata.py b/src/modules/metadata.py index 1787bfb..d322cf1 100644 --- a/src/modules/metadata.py +++ b/src/modules/metadata.py @@ -3,7 +3,7 @@ from string import Template from typing import Callable, Iterable, Iterator, List, Match, NamedTuple, Optional, Pattern, Tuple, TypeVar, Union -from modules.utilities import first, isSomething, second +from modules.utilities import first, second class MetadataError(Exception): def __init__(self,*args,**kwargs): @@ -97,10 +97,10 @@ def tag(name: str) -> str: def isWhitespaceLine(s: str) -> bool: - return isSomething(re.compile(r"^\s*$").match(s)) + return re.compile(r"^\s*$").match(s) is not None def isCommentLine(s: str) -> bool: - return isSomething(re.compile(r"^\s*" + PREFIX_COMMENT + r".*$").match(s)) + return re.compile(r"^\s*" + PREFIX_COMMENT + r".*$").match(s) is not None def extract(userscriptContent: str) -> str: # raises MetadataError match_metadataBlock: Optional[Match] = REGEX_METADATA_BLOCK.search(userscriptContent) diff --git a/src/modules/patterns.py b/src/modules/patterns.py index ac02db5..cdb15d0 100644 --- a/src/modules/patterns.py +++ b/src/modules/patterns.py @@ -1,7 +1,7 @@ import re from typing import Match, Optional, Pattern -from modules.utilities import first, isSomething +from modules.utilities import first REGEX_MATCH_ALL = r"" REGEX_MATCH_SCHEME = r"\*|https?" @@ -34,15 +34,15 @@ def normalizeMatchPattern(pattern: str) -> str: def isMatchPattern(pattern: str) -> bool: - return isSomething(REGEX_MATCH_PATTERN.match(pattern)) + return REGEX_MATCH_PATTERN.match(pattern) is not None def isIncludePattern(pattern: str) -> bool: - return isSomething(REGEX_INCLUDE_PATTERN.match(pattern)) + return REGEX_INCLUDE_PATTERN.match(pattern) is not None def isIncludePattern_regex(pattern: str) -> bool: - return isSomething(re.compile(REGEX_INCLUDE_REGEX).match(pattern)) + return re.compile(REGEX_INCLUDE_REGEX).match(pattern) is not None def withoutSurroundingSlashes(s: str) -> str: diff --git a/src/modules/userscript.py b/src/modules/userscript.py index 00cf58a..7cbd328 100644 --- a/src/modules/userscript.py +++ b/src/modules/userscript.py @@ -9,7 +9,7 @@ import modules.metadata as metadata from modules.metadata import Metadata, Tag, Tag_boolean, Tag_string from modules.patterns import isIncludePattern, isMatchPattern, regexFromIncludePattern -from modules.utilities import compose2, isSomething, stripIndentation, strs +from modules.utilities import compose2, stripIndentation, strs class UserscriptError(Exception): def __init__(self,*args,**kwargs): @@ -84,7 +84,7 @@ def __init__(self,*args,**kwargs): unique = True, default = None, required = False, - predicate = lambda val: isSomething(REGEX_URL.match(val)), + predicate = lambda val: REGEX_URL.match(val) is not None, ) METADATA_TAGS: List[Tag] = [ @@ -133,14 +133,14 @@ def create(content: str) -> Userscript: valueOf = metadata.valueGetter_one(validMetadata) allValuesOf = metadata.valueGetter_all(validMetadata) includePatternRegexes: List[Pattern] = list(filter( - isSomething, + lambda x: x is not None, map( compose2(regexFromIncludePattern_safe, str), allValuesOf(tag_include) ) )) excludePatternRegexes: List[Pattern] = list(filter( - isSomething, + lambda x: x is not None, map( compose2(regexFromIncludePattern_safe, str), allValuesOf(tag_exclude) @@ -163,10 +163,10 @@ def create(content: str) -> Userscript: def applicableChecker(url: str) -> Callable[[Userscript], bool]: def isApplicable(userscript: Userscript) -> bool: for regex in userscript.excludePatternRegexes: - if isSomething(regex.search(url)): + if regex.search(url) is not None: return False for regex in userscript.includePatternRegexes: - if isSomething(regex.search(url)): + if regex.search(url) is not None: return True for pattern in userscript.matchPatterns: if urlmatch(pattern, url): diff --git a/src/modules/utilities.py b/src/modules/utilities.py index cb465f1..7a7628c 100644 --- a/src/modules/utilities.py +++ b/src/modules/utilities.py @@ -23,10 +23,6 @@ def second(tuple: Tuple[A, B]) -> B: return b -def isSomething(x: Optional[A]) -> bool: - return x is not None - - def strs(xs: Any) -> List[str]: return list(map(str, xs)) From bba90376ce0d0e46a22a3c55301609f664433e44 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 22 Jun 2025 16:04:55 +0200 Subject: [PATCH 090/137] Remove unused UserscriptError class As far as I can tell, it has never been used, in the sense that no instances have ever been constructed. See for example `git log -p -S UserscriptError`. --- src/injector.py | 6 +----- src/modules/userscript.py | 4 ---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/injector.py b/src/injector.py index 8db983c..9ce017c 100644 --- a/src/injector.py +++ b/src/injector.py @@ -17,7 +17,7 @@ from modules.requests import CONTENT_TYPE, containsQueryParam, inferEncoding import modules.text as T import modules.userscript as userscript -from modules.userscript import Userscript, UserscriptError +from modules.userscript import Userscript from modules.utilities import first, flag, fromOptional, itemList, second PATTERN_USERSCRIPT: str = "*.user.js" @@ -115,10 +115,6 @@ def loadUserscripts(directory: str) -> List[Userscript]: logError("Metadata error:") logError(str(err)) continue - except UserscriptError as err: - logError("Userscript error:") - logError(str(err)) - continue os.chdir(workingDirectory) # so mitmproxy does not unload the script logInfo("") logInfo(str(len(loadedUserscripts)) + " userscript(s) loaded:") diff --git a/src/modules/userscript.py b/src/modules/userscript.py index 7cbd328..7b60d87 100644 --- a/src/modules/userscript.py +++ b/src/modules/userscript.py @@ -11,10 +11,6 @@ from modules.patterns import isIncludePattern, isMatchPattern, regexFromIncludePattern from modules.utilities import compose2, stripIndentation, strs -class UserscriptError(Exception): - def __init__(self,*args,**kwargs): - Exception.__init__(self,*args,**kwargs) - REGEX_URL: Pattern = re.compile(r"^https?://") directive_name : str = "name" From 015f2ccc68420c06e4e7df133efc38b72864181f Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 13 Jul 2025 23:01:27 +0200 Subject: [PATCH 091/137] Remove support for typechecking individual files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It makes the typechecking script more complicated, and I never use it. 💡 `git show --ignore-all-space` --- typecheck | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/typecheck b/typecheck index a5bfa45..7adae6b 100755 --- a/typecheck +++ b/typecheck @@ -3,9 +3,5 @@ export MYPYPATH=/usr/local/lib/python3.6/dist-packages/ cd src -if [ "$1" == "" ]; then - mypy *.py --ignore-missing-imports --follow-imports skip - mypy modules/*.py --ignore-missing-imports --follow-imports skip -else - mypy $1 --ignore-missing-imports --follow-imports skip -fi +mypy *.py --ignore-missing-imports --follow-imports skip +mypy modules/*.py --ignore-missing-imports --follow-imports skip From a07173bc17ba596c20b22bd6a563bafbf3af8894 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 13 Jul 2025 23:03:41 +0200 Subject: [PATCH 092/137] Don't cd into `src/` in typechecking script It took me a while to realize that the `mypy` commands in the typechecking script didn't run in the repo root, but instead in `src/`. --- typecheck | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/typecheck b/typecheck index 7adae6b..870429d 100755 --- a/typecheck +++ b/typecheck @@ -2,6 +2,5 @@ export MYPYPATH=/usr/local/lib/python3.6/dist-packages/ -cd src -mypy *.py --ignore-missing-imports --follow-imports skip -mypy modules/*.py --ignore-missing-imports --follow-imports skip +mypy src/*.py --ignore-missing-imports --follow-imports skip +mypy src/modules/*.py --ignore-missing-imports --follow-imports skip From 3399b69e0f2e29e4a6d9f2a470f004736d6e7798 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 13 Jul 2025 23:11:49 +0200 Subject: [PATCH 093/137] Make typechecking fail on errors outside `src/modules/` Today, this change doesn't cause `make` to fail: ```diff --- a/src/launcher.py +++ b/src/launcher.py @@ -15,4 +15,5 @@ import modules.text as T from modules.utilities import flag, idem, itemList +foo: str = 5 FILENAME_INJECTOR: str = "injector.py" MATCH_NO_HOSTS = r"^$" ``` The first `mypy` command in the script fails, but without `-e`, the script just proceeds to the second `mypy` command, which succeeds. This commit makes the script fail as expected. --- typecheck | 2 ++ 1 file changed, 2 insertions(+) diff --git a/typecheck b/typecheck index 870429d..12be8aa 100755 --- a/typecheck +++ b/typecheck @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + export MYPYPATH=/usr/local/lib/python3.6/dist-packages/ mypy src/*.py --ignore-missing-imports --follow-imports skip From 0176bc7cfe611ff68e0ad1544260238f200fc193 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sat, 9 Aug 2025 10:45:34 +0200 Subject: [PATCH 094/137] Upgrade to Python 3.9 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 120681b..42793fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.10-slim AS base +FROM python:3.9.23-slim AS base FROM base AS builder From 377caf60f2f6afb595cce75faeb29603e06dba76 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sat, 9 Aug 2025 10:55:32 +0200 Subject: [PATCH 095/137] Update Compose file image version in release workflow The Compose file template was added in ef6c883445042af49a940b3396ae22c775076fed. --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bb75a06..294001e 100644 --- a/Makefile +++ b/Makefile @@ -27,9 +27,11 @@ ifeq "$(TAG)" "$(DEFAULT_TAG)" endif # Update in-app version: sed -i 's/^VERSION: str = "[^"]*"/VERSION: str = "$(TAG)"/' $(FILE_WITH_VERSION) +# Update readme version: + sed -i 's#image: alling/userscript-proxy:.*#image: alling/userscript-proxy:$(TAG)#' README.md docker build -t $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) . docker tag $(DOCKER_USER)/$(DOCKER_REPO):$(TAG) $(DOCKER_USER)/$(DOCKER_REPO):$(DEFAULT_TAG) - git add $(FILE_WITH_VERSION) + git add $(FILE_WITH_VERSION) README.md git commit -m "v$(TAG)" git tag "v$(TAG)" From b0c4a262137272036e028782f53084e9a6062e1b Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sat, 9 Aug 2025 11:16:00 +0200 Subject: [PATCH 096/137] v1.1.1 --- README.md | 2 +- src/modules/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 38401ba..058c022 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ If you're familiar with Userscript Proxy, you might want to use Docker Compose: ```yaml services: userscript-proxy: - image: alling/userscript-proxy:1.1.0 + image: alling/userscript-proxy:1.1.1 container_name: userscript-proxy command: - --userscripts-dir diff --git a/src/modules/constants.py b/src/modules/constants.py index 264c6a4..c6970d7 100644 --- a/src/modules/constants.py +++ b/src/modules/constants.py @@ -1,6 +1,6 @@ VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" -VERSION: str = "1.1.0" +VERSION: str = "1.1.1" ATTRIBUTE_UP_VERSION: str = "data-userscript-proxy-version" DEFAULT_RULES_DIR: str = "default-rules/" From 31eb23c7247a064b2392f32966bb03281264ecda Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sat, 9 Aug 2025 11:22:41 +0200 Subject: [PATCH 097/137] Document how to push to Docker Hub The reason it's not multiple `echo` commands is that `make` prints each command, resulting in completely unreadable output. --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 294001e..14da995 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,7 @@ endif git add $(FILE_WITH_VERSION) README.md git commit -m "v$(TAG)" git tag "v$(TAG)" + echo "Run these commands to push to Docker Hub:\n\n docker push alling/userscript-proxy:$(TAG)\n docker push alling/userscript-proxy:$(DEFAULT_TAG)\n" start: image # The -t flag enables colored output: From 4d6a0fdd59120daf06b2418ac1f630f19de415de Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 13 Jul 2025 18:32:28 +0200 Subject: [PATCH 098/137] Use native `list` instead of `typing.List` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [PEP 585], included in Python 3.9, "enable[d] support for the generics syntax in all standard collections currently available in the `typing` module." This means that we can use `list` instead of importing `List` from the `typing` module. The intention is to make other analogous changes, such as replacing `typing.Tuple` with `tuple`, in future commits. [PEP 585]: https://peps.python.org/pep-0585/ 💡 `git show --color-words='\w+|.'` --- src/injector.py | 14 +++++++------- src/launcher.py | 7 +++---- src/modules/csp.py | 4 ++-- src/modules/ignore.py | 4 ++-- src/modules/inline.py | 6 +++--- src/modules/metadata.py | 16 ++++++++-------- src/modules/userscript.py | 16 ++++++++-------- src/modules/utilities.py | 4 ++-- 8 files changed, 35 insertions(+), 36 deletions(-) diff --git a/src/injector.py b/src/injector.py index 9ce017c..c01f185 100644 --- a/src/injector.py +++ b/src/injector.py @@ -2,7 +2,7 @@ import glob import os import shlex -from typing import Callable, Iterable, List, Optional, Tuple +from typing import Callable, Iterable, Optional, Tuple from bs4 import BeautifulSoup, Comment, Doctype from mitmproxy import ctx, http @@ -21,7 +21,7 @@ from modules.utilities import first, flag, fromOptional, itemList, second PATTERN_USERSCRIPT: str = "*.user.js" -RELEVANT_CONTENT_TYPES: List[str] = ["text/html", "application/xhtml+xml"] +RELEVANT_CONTENT_TYPES: list[str] = ["text/html", "application/xhtml+xml"] CHARSET_DEFAULT: str = "utf-8" TAB: str = " " LIST_ITEM_PREFIX: str = TAB + "• " @@ -85,8 +85,8 @@ def option(key: str): return ctx.options.__getattr__(sanitize(key)) -def loadUserscripts(directory: str) -> List[Userscript]: - loadedUserscripts: List[Tuple[Userscript, str]] = [] +def loadUserscripts(directory: str) -> list[Userscript]: + loadedUserscripts: list[Tuple[Userscript, str]] = [] workingDirectory = os.getcwd() logInfo(f"""Looking recursively for userscripts ({PATTERN_USERSCRIPT}) in directory `{directory}` ...""") os.chdir(directory) @@ -129,7 +129,7 @@ def loadUserscripts(directory: str) -> List[Userscript]: class UserscriptInjector: def __init__(self): - self.userscripts: List[Userscript] = [] + self.userscripts: list[Userscript] = [] def load(self, loader): @@ -165,7 +165,7 @@ def response(self, flow: http.HTTPFlow): if CONTENT_TYPE in response.headers: if any(map(lambda t: t in response.headers[CONTENT_TYPE], RELEVANT_CONTENT_TYPES)): # Response is a web page; proceed. - injections: List[csp.Injection] = [] + injections: list[csp.Injection] = [] soup = BeautifulSoup( response.content, HTML_PARSER, @@ -217,7 +217,7 @@ def response(self, flow: http.HTTPFlow): ) -def handleContentSecurityPolicy(response: http.HTTPFlow.response, injections: List[csp.Injection]): +def handleContentSecurityPolicy(response: http.HTTPFlow.response, injections: list[csp.Injection]): # If there is a CSP header, we may need to modify it for the userscript(s) to work. ContentSecurityPolicy = "Content-Security-Policy" if ContentSecurityPolicy in response.headers: diff --git a/src/launcher.py b/src/launcher.py index 7eebc9b..7c0b21f 100755 --- a/src/launcher.py +++ b/src/launcher.py @@ -4,7 +4,6 @@ import os import shlex import subprocess -from typing import List from modules.argparser import getArgparser import modules.arguments as A @@ -22,7 +21,7 @@ def printInfo( useFiltering: bool, useIntercept: bool, useTransparent: bool, - filterRules: List[str], + filterRules: list[str], ): print() print("mitmproxy will be run in " + ("TRANSPARENT" if useTransparent else "REGULAR") + " mode.") @@ -62,7 +61,7 @@ def ruleFilesContent_default(): if useDefaultRules: print(f"Reading default {'intercept' if useIntercept else 'ignore'} rules ...") globPatternForDefaultRules = C.DEFAULT_INTERCEPT_RULES if useIntercept else C.DEFAULT_IGNORE_RULES - filenames: List[str] = [ shlex.quote(unsafeFilename) for unsafeFilename in glob.glob(globPatternForDefaultRules) ] + filenames: list[str] = [ shlex.quote(unsafeFilename) for unsafeFilename in glob.glob(globPatternForDefaultRules) ] acc = "" for filename in filenames: print("Reading " + filename + " ...") @@ -73,7 +72,7 @@ def ruleFilesContent_default(): def ruleFilesContent_custom(): if useCustomFiltering: print(f"Reading custom {'intercept' if useIntercept else 'ignore'} rules ({globPattern}) ...") - filenames: List[str] = [ shlex.quote(unsafeFilename) for unsafeFilename in glob.glob(globPattern) ] + filenames: list[str] = [ shlex.quote(unsafeFilename) for unsafeFilename in glob.glob(globPattern) ] acc = "" for filename in filenames: print("Reading " + filename + " ...") diff --git a/src/modules/csp.py b/src/modules/csp.py index 230eb98..36221f4 100644 --- a/src/modules/csp.py +++ b/src/modules/csp.py @@ -1,5 +1,5 @@ import secrets -from typing import List, NamedTuple, Optional +from typing import NamedTuple, Optional from modules.userscript import Userscript @@ -11,7 +11,7 @@ class Injection(NamedTuple): nonce: Optional[str] -def headerWithScriptsAllowed(cspHeaderValue: str, injections: List[Injection]) -> str: +def headerWithScriptsAllowed(cspHeaderValue: str, injections: list[Injection]) -> str: # Example CSP header: # # Content-Security-Policy: default-src 'self'; frame-src 'self'; img-src https:; connect-src 'self' diff --git a/src/modules/ignore.py b/src/modules/ignore.py index 7eb66b3..be02fcb 100644 --- a/src/modules/ignore.py +++ b/src/modules/ignore.py @@ -1,5 +1,5 @@ import re -from typing import List, Pattern +from typing import Pattern from modules.patterns import isIncludePattern_regex, regexify, withoutSurroundingSlashes @@ -8,7 +8,7 @@ PIPE: str = "|" REGEX_COMMENT: Pattern = re.compile(r"\#.*$") -def rulesIn(text: str) -> List[str]: +def rulesIn(text: str) -> list[str]: return list(filter( lambda s: s != "", map(withoutCommentAndTrimmed, text.splitlines()) diff --git a/src/modules/inline.py b/src/modules/inline.py index 58bda01..99166b8 100644 --- a/src/modules/inline.py +++ b/src/modules/inline.py @@ -1,9 +1,9 @@ import re -from typing import List, Pattern +from typing import Pattern # https://www.w3.org/TR/html/semantics-scripting.html#script-content-restrictions -DANGEROUS_SEQUENCES: List[str] = [ +DANGEROUS_SEQUENCES: list[str] = [ r"