diff --git a/README.md b/README.md index 4c11792..91c10af 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,157 @@ -# userscript-proxy -MITM userscript injector +# Userscript Proxy + +Browser extensions on iOS, Android and pretty much any other web browsing device. +No jailbreak/root required. + +Userscript Proxy is built around [mitmproxy](mitmproxy) and acts as a MITM, injecting matching userscripts into web pages as they flow through it. +Both HTTP and HTTPS are supported. + + +## Security + +**Userscript Proxy can (and must be able to) read and modify all HTTP(S) traffic** sent to and from the device in question, so the only reasonably secure way to use it is to run it on a server controlled by oneself. + + +## Ignoring hosts + +Apps like App Store and Facebook Messenger refuse to connect through a MITM proxy, so their traffic must be ignored by mitmproxy. +There are two approaches: + + * Blacklisting hosts that cannot connect through the proxy. + Tedious, because you have to add exceptions for apps and such all the time. + * Whitelisting hosts where userscripts should be applied. + Works well in general, but does not allow universal userscripts that run on all sites, and the whitelist must be updated when a new userscript is added. + +Blacklisting or whitelisting is done by giving the `--ignore` or `--intercept` flag together with one or more files containing **ignore/intercept rules**. +Examples: + +```bash +# Take ignore rules from ignore.txt (included): +python3.6 launcher.py --ignore "ignore.txt" + +# Take intercept rules from all .txt files whose names start with "foo": +python3.6 launcher.py --intercept "foo*.txt" +``` + +Rules can be specified in two ways: + +### Basic pattern + +Based on the syntax used by userscript `@include` directives. +Asterisk (`*`) means any string (including the empty string). +`*.` is automatically prepended. +`:*` is automatically appended unless the rule contains a colon (`:`). + +To match a domain without matching all of its subdomains, use a regex rule instead (see below). + +#### Examples + +| Rule | Matches | +|----------------|-----------------------------------------------------------------| +| `site.com` | `site.com` and `x.site.com` | +| `api.site.com` | `api.site.com` and `x.api.site.com`, but not `www.site.com` | +| `*cdn.net` | `cdn.net`, `fbcdn.net` and `x.fbcdn.net`, but not `cdn.net.com` | +| `site.com:80` | `site.com:80` and `x.site.com:80`, but not `site.com:443` | + +### Regular expression + +If a rule starts and ends with a slash (`/`), it is treated as a Python regex. + +Note that the string to match against contains both a host and a port, e.g. `example.com:443`, and that the regex is used verbatim (i.e. you have to explicitly provide `^` etc if desired). +The only exception is that the case-insensitivity flag (`?i`) is automatically added. + +Also, be careful with `$`: A regex like `/site.com$/` will never match, because it will only be used to check strings like `site.com:80`. + +Anything from a `#` until the end of the line is treated as a comment. +Leading and trailing whitespace have no effect. + +#### Examples + +| Rule | Matches | +|-----------------|-------------------------------------------------------------------| +| `/cdn\./` | `fbcdn.net`, `cdn.site.com`, `cdn.x.site.com`, but not `cdna.com` | +| `/^site\.com:/` | `site.com`, but not `x.site.com`, `mysite.com` or `site.com.net` | + + +## Data usage + +Userscript Proxy has no data usage impact when no userscript is injected, i.e. for URLs without any matching userscript. +When a script _is_ injected, **exactly one** of the following things happens: + + * The entire userscript is injected as inline JavaScript (potentially dozens or even hundreds of kilobytes). + * A ``), never linked (``). +Useful to test new userscript features without having to re-upload the userscript and clear browser cache. + +### `--list-injected`, `-l` + +Insert an HTML comment in each page specifying which userscripts (if any) were injected. + +### `--port PORT`, `-p PORT` + +Make mitmproxy listen to TCP port `PORT`. +Defaults to `8080`. + +### `--query-param-to-disable PARAM`, `-q PARAM` + +Disable userscripts when the request URL contains `PARAM` as a query parameter. +For example, use `-q foo` to disable userscripts for `http://example.com?foo`. +Defaults to `nouserscripts`. + +### `--recursive`, `-r` + +Recurse into directories when looking for userscripts. + +### `--transparent`, `-t` + +Run mitmproxy in [transparent mode](transparent-mode). +Useful if you cannot set a proxy in the client, e.g. when using OpenVPN Connect on Android to connect to a VPN server on the network where your proxy is running. +In such cases, you have to route traffic from the client to the proxy at the network layer instead, making transparent mode necessary. + +### `--userscripts DIR`, `-u DIR` + +Load userscripts from directory `DIR`. +Defaults to `userscripts`. + + +[mitmproxy]: https://mitmproxy.org +[minification]: https://en.wikipedia.org/wiki/Minification_(programming) +[metadata]: https://wiki.greasespot.net/Metadata_Block +[transparent-mode]: https://docs.mitmproxy.org/stable/concepts-modes/#transparent-proxy +[gm-api]: https://wiki.greasespot.net/GM.getValue diff --git a/ignore.txt b/ignore.txt new file mode 100644 index 0000000..c556106 --- /dev/null +++ b/ignore.txt @@ -0,0 +1,39 @@ +# See README.md for information about the syntax used in this file. + +# Applications that cannot connect through mitmproxy: + itunes.apple.com # App Store + xp.apple.com # App Store + apps.apple.com # App Store + mzstatic.com # App Store + graph.facebook.com # Messenger + api.facebook.com # Messenger + edge-mqtt.facebook.com # Messenger + edge-chat.facebook.com # Messenger + slack.com # Slack + api*.dropbox.com # Dropbox app + bolt.dropbox.com # Dropbox app + ls.apple.com # Apple services + icloud.com # Apple services + crashlytics.com # Crashlytics error reporting + twimg.com # Twitter + ap.spotify.com # Spotify app + wg.spotify.com # Spotify app + api.branch.io # Branch™ deep links + ggpht.com # Google/Android + www.google.com # Google/Android + gvt1.com # Google/Android + +# Traffic irrelevant to Userscript Proxy: + /cdn\./ + akamai*.net # Akamai CDN + googleusercontent.com # Google CDN + clients*.google.com # Google Maps etc + googleapis.com # Google APIs + i.ytimg.com # YouTube thumbnails + googlevideo.com # YouTube video content + s.youtube.com # YouTube stats + api.twitch.tv # Twitch app metadata + ttvnw.net # Twitch video content + /^149\.154\.16[4-7]\.\d+:/ # Telegram Messenger Network + audio-fa.spotify.com # Spotify audio content + slack-msgs.com # Slack app diff --git a/injector.py b/injector.py new file mode 100644 index 0000000..d88553f --- /dev/null +++ b/injector.py @@ -0,0 +1,212 @@ +from typing import Optional, Iterable, List, Callable, Pattern, Match, Tuple +import glob, os +from bs4 import BeautifulSoup, Comment, Doctype +from mitmproxy import ctx, http +from functools import partial +import shlex +import warnings +from modules.metadata import MetadataError, PREFIX_TAG +import modules.userscript as userscript +import modules.inline as inline +import modules.text as T +from modules.userscript import Userscript, UserscriptError, document_end, document_start, document_idle +from modules.utilities import first, second, itemList, fromOptional, flag, idem +from modules.constants import VERSION, VERSION_PREFIX, APP_NAME, DEFAULT_USERSCRIPTS_DIR, DEFAULT_QUERY_PARAM_TO_DISABLE +from modules.inject import Options, inject +from modules.misc import sanitize +from modules.requests import CONTENT_TYPE, inferEncoding, requestContainsQueryParam + +PATTERN_USERSCRIPT: str = "*.user.js" +RELEVANT_CONTENT_TYPES: List[str] = ["text/html", "application/xhtml+xml"] +CHARSET_DEFAULT: str = "utf-8" +TAB: str = " " +LIST_ITEM_PREFIX: str = TAB + "• " +HTML_PARSER: str = "lxml" +# lxml handles non-uppercase DOCTYPE correctly; html.parser does not: It emits +# if the original source code contained . +HTML_INFO_COMMENT_PREFIX: str = f""" +[{T.INFO_MESSAGE}] +""" + + +def logInfo(s: str) -> None: + try: + ctx.log.info(s) + except Exception: + print(s) + +def logWarning(s: str) -> None: + try: + ctx.log.warn(s) + except Exception: + print(s) + +def logError(s: str) -> None: + try: + ctx.log.error(s) + except Exception: + print(s) + +def indexOfDTD(soup: BeautifulSoup) -> Optional[int]: + index: int = 0 + for item in soup.contents: + if isinstance(item, Doctype): + return index + index += 1 + return None + +bulletList: Callable[[Iterable[str]], str] = partial(itemList, LIST_ITEM_PREFIX) + +def unsafeSequencesMessage(script: Userscript) -> str: + sequences = script.unsafeSequences + return f"""{script.name} cannot be injected because it contains {"these unsafe sequences" if len(sequences) > 1 else "this unsafe sequence"}: + +{itemList(TAB, sequences)} + +