diff --git a/README.md b/README.md
index 4c11792..91c10af 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,157 @@
-# userscript-proxy
-MITM userscript injector
+# Userscript Proxy
+
+Browser extensions on iOS, Android and pretty much any other web browsing device.
+No jailbreak/root required.
+
+Userscript Proxy is built around [mitmproxy](mitmproxy) and acts as a MITM, injecting matching userscripts into web pages as they flow through it.
+Both HTTP and HTTPS are supported.
+
+
+## Security
+
+**Userscript Proxy can (and must be able to) read and modify all HTTP(S) traffic** sent to and from the device in question, so the only reasonably secure way to use it is to run it on a server controlled by oneself.
+
+
+## Ignoring hosts
+
+Apps like App Store and Facebook Messenger refuse to connect through a MITM proxy, so their traffic must be ignored by mitmproxy.
+There are two approaches:
+
+ * Blacklisting hosts that cannot connect through the proxy.
+ Tedious, because you have to add exceptions for apps and such all the time.
+ * Whitelisting hosts where userscripts should be applied.
+ Works well in general, but does not allow universal userscripts that run on all sites, and the whitelist must be updated when a new userscript is added.
+
+Blacklisting or whitelisting is done by giving the `--ignore` or `--intercept` flag together with one or more files containing **ignore/intercept rules**.
+Examples:
+
+```bash
+# Take ignore rules from ignore.txt (included):
+python3.6 launcher.py --ignore "ignore.txt"
+
+# Take intercept rules from all .txt files whose names start with "foo":
+python3.6 launcher.py --intercept "foo*.txt"
+```
+
+Rules can be specified in two ways:
+
+### Basic pattern
+
+Based on the syntax used by userscript `@include` directives.
+Asterisk (`*`) means any string (including the empty string).
+`*.` is automatically prepended.
+`:*` is automatically appended unless the rule contains a colon (`:`).
+
+To match a domain without matching all of its subdomains, use a regex rule instead (see below).
+
+#### Examples
+
+| Rule | Matches |
+|----------------|-----------------------------------------------------------------|
+| `site.com` | `site.com` and `x.site.com` |
+| `api.site.com` | `api.site.com` and `x.api.site.com`, but not `www.site.com` |
+| `*cdn.net` | `cdn.net`, `fbcdn.net` and `x.fbcdn.net`, but not `cdn.net.com` |
+| `site.com:80` | `site.com:80` and `x.site.com:80`, but not `site.com:443` |
+
+### Regular expression
+
+If a rule starts and ends with a slash (`/`), it is treated as a Python regex.
+
+Note that the string to match against contains both a host and a port, e.g. `example.com:443`, and that the regex is used verbatim (i.e. you have to explicitly provide `^` etc if desired).
+The only exception is that the case-insensitivity flag (`?i`) is automatically added.
+
+Also, be careful with `$`: A regex like `/site.com$/` will never match, because it will only be used to check strings like `site.com:80`.
+
+Anything from a `#` until the end of the line is treated as a comment.
+Leading and trailing whitespace have no effect.
+
+#### Examples
+
+| Rule | Matches |
+|-----------------|-------------------------------------------------------------------|
+| `/cdn\./` | `fbcdn.net`, `cdn.site.com`, `cdn.x.site.com`, but not `cdna.com` |
+| `/^site\.com:/` | `site.com`, but not `x.site.com`, `mysite.com` or `site.com.net` |
+
+
+## Data usage
+
+Userscript Proxy has no data usage impact when no userscript is injected, i.e. for URLs without any matching userscript.
+When a script _is_ injected, **exactly one** of the following things happens:
+
+ * The entire userscript is injected as inline JavaScript (potentially dozens or even hundreds of kilobytes).
+ * A ``), never linked (``).
+Useful to test new userscript features without having to re-upload the userscript and clear browser cache.
+
+### `--list-injected`, `-l`
+
+Insert an HTML comment in each page specifying which userscripts (if any) were injected.
+
+### `--port PORT`, `-p PORT`
+
+Make mitmproxy listen to TCP port `PORT`.
+Defaults to `8080`.
+
+### `--query-param-to-disable PARAM`, `-q PARAM`
+
+Disable userscripts when the request URL contains `PARAM` as a query parameter.
+For example, use `-q foo` to disable userscripts for `http://example.com?foo`.
+Defaults to `nouserscripts`.
+
+### `--recursive`, `-r`
+
+Recurse into directories when looking for userscripts.
+
+### `--transparent`, `-t`
+
+Run mitmproxy in [transparent mode](transparent-mode).
+Useful if you cannot set a proxy in the client, e.g. when using OpenVPN Connect on Android to connect to a VPN server on the network where your proxy is running.
+In such cases, you have to route traffic from the client to the proxy at the network layer instead, making transparent mode necessary.
+
+### `--userscripts DIR`, `-u DIR`
+
+Load userscripts from directory `DIR`.
+Defaults to `userscripts`.
+
+
+[mitmproxy]: https://mitmproxy.org
+[minification]: https://en.wikipedia.org/wiki/Minification_(programming)
+[metadata]: https://wiki.greasespot.net/Metadata_Block
+[transparent-mode]: https://docs.mitmproxy.org/stable/concepts-modes/#transparent-proxy
+[gm-api]: https://wiki.greasespot.net/GM.getValue
diff --git a/ignore.txt b/ignore.txt
new file mode 100644
index 0000000..c556106
--- /dev/null
+++ b/ignore.txt
@@ -0,0 +1,39 @@
+# See README.md for information about the syntax used in this file.
+
+# Applications that cannot connect through mitmproxy:
+ itunes.apple.com # App Store
+ xp.apple.com # App Store
+ apps.apple.com # App Store
+ mzstatic.com # App Store
+ graph.facebook.com # Messenger
+ api.facebook.com # Messenger
+ edge-mqtt.facebook.com # Messenger
+ edge-chat.facebook.com # Messenger
+ slack.com # Slack
+ api*.dropbox.com # Dropbox app
+ bolt.dropbox.com # Dropbox app
+ ls.apple.com # Apple services
+ icloud.com # Apple services
+ crashlytics.com # Crashlytics error reporting
+ twimg.com # Twitter
+ ap.spotify.com # Spotify app
+ wg.spotify.com # Spotify app
+ api.branch.io # Branch™ deep links
+ ggpht.com # Google/Android
+ www.google.com # Google/Android
+ gvt1.com # Google/Android
+
+# Traffic irrelevant to Userscript Proxy:
+ /cdn\./
+ akamai*.net # Akamai CDN
+ googleusercontent.com # Google CDN
+ clients*.google.com # Google Maps etc
+ googleapis.com # Google APIs
+ i.ytimg.com # YouTube thumbnails
+ googlevideo.com # YouTube video content
+ s.youtube.com # YouTube stats
+ api.twitch.tv # Twitch app metadata
+ ttvnw.net # Twitch video content
+ /^149\.154\.16[4-7]\.\d+:/ # Telegram Messenger Network
+ audio-fa.spotify.com # Spotify audio content
+ slack-msgs.com # Slack app
diff --git a/injector.py b/injector.py
new file mode 100644
index 0000000..d88553f
--- /dev/null
+++ b/injector.py
@@ -0,0 +1,212 @@
+from typing import Optional, Iterable, List, Callable, Pattern, Match, Tuple
+import glob, os
+from bs4 import BeautifulSoup, Comment, Doctype
+from mitmproxy import ctx, http
+from functools import partial
+import shlex
+import warnings
+from modules.metadata import MetadataError, PREFIX_TAG
+import modules.userscript as userscript
+import modules.inline as inline
+import modules.text as T
+from modules.userscript import Userscript, UserscriptError, document_end, document_start, document_idle
+from modules.utilities import first, second, itemList, fromOptional, flag, idem
+from modules.constants import VERSION, VERSION_PREFIX, APP_NAME, DEFAULT_USERSCRIPTS_DIR, DEFAULT_QUERY_PARAM_TO_DISABLE
+from modules.inject import Options, inject
+from modules.misc import sanitize
+from modules.requests import CONTENT_TYPE, inferEncoding, requestContainsQueryParam
+
+PATTERN_USERSCRIPT: str = "*.user.js"
+RELEVANT_CONTENT_TYPES: List[str] = ["text/html", "application/xhtml+xml"]
+CHARSET_DEFAULT: str = "utf-8"
+TAB: str = " "
+LIST_ITEM_PREFIX: str = TAB + "• "
+HTML_PARSER: str = "lxml"
+# lxml handles non-uppercase DOCTYPE correctly; html.parser does not: It emits
+# if the original source code contained .
+HTML_INFO_COMMENT_PREFIX: str = f"""
+[{T.INFO_MESSAGE}]
+"""
+
+
+def logInfo(s: str) -> None:
+ try:
+ ctx.log.info(s)
+ except Exception:
+ print(s)
+
+def logWarning(s: str) -> None:
+ try:
+ ctx.log.warn(s)
+ except Exception:
+ print(s)
+
+def logError(s: str) -> None:
+ try:
+ ctx.log.error(s)
+ except Exception:
+ print(s)
+
+def indexOfDTD(soup: BeautifulSoup) -> Optional[int]:
+ index: int = 0
+ for item in soup.contents:
+ if isinstance(item, Doctype):
+ return index
+ index += 1
+ return None
+
+bulletList: Callable[[Iterable[str]], str] = partial(itemList, LIST_ITEM_PREFIX)
+
+def unsafeSequencesMessage(script: Userscript) -> str:
+ sequences = script.unsafeSequences
+ return f"""{script.name} cannot be injected because it contains {"these unsafe sequences" if len(sequences) > 1 else "this unsafe sequence"}:
+
+{itemList(TAB, sequences)}
+
+ List[str]:
+ return re.findall(REGEX, source)
diff --git a/metadata.py b/modules/metadata.py
similarity index 68%
rename from metadata.py
rename to modules/metadata.py
index b3e415a..773188e 100644
--- a/metadata.py
+++ b/modules/metadata.py
@@ -1,9 +1,9 @@
-from typing import TypeVar, Tuple, List, Iterator, Pattern, Match, Optional, Union, Callable, NamedTuple
+from typing import TypeVar, Tuple, List, Iterator, Iterable, Pattern, Match, Optional, Union, Callable, NamedTuple
import re
from string import Template
from functools import reduce
-from utilities import A, B, first, second, isSomething
-import warnings
+import itertools
+from modules.utilities import first, second, isSomething
class MetadataError(Exception):
def __init__(self,*args,**kwargs):
@@ -39,21 +39,21 @@ class Tag_boolean(NamedTuple):
BLOCK_START: str = "==UserScript=="
BLOCK_END: str = "==/UserScript=="
-REGEX_EMPTY_LINE_COMMENT: Pattern = re.compile(r"^(?:\/\/)?\s*$")
+REGEXGROUP_CONTENT: str = "content"
REGEX_METADATA_BLOCK: Pattern = re.compile(
PREFIX_COMMENT + r"\s*" + BLOCK_START + r"\n"
- + r"(.*)"
+ + r"(?P<" + REGEXGROUP_CONTENT + r">.*)"
+ PREFIX_COMMENT + r"\s*" + BLOCK_END,
re.DOTALL
)
-INDEX_GROUP_BLOCK_CONTENT: int = 1
+REGEXGROUP_TAGNAME: str = "tagname"
+REGEXGROUP_TAGVALUE: str = "tagvalue"
REGEX_METADATA_LINE: Pattern = re.compile(
r"^\s*" + PREFIX_COMMENT
+ r"\s*" + PREFIX_TAG
- + r"([^\s]+)(?:\s+?(\S.*)?)?$"
+ + r"(?P<" + REGEXGROUP_TAGNAME + r">[^\s]+)"
+ + r"(?:\s+?(?P<" + REGEXGROUP_TAGVALUE + r">\S.*)?)?$"
)
-INDEX_GROUP_TAGNAME: int = 1
-INDEX_GROUP_TAGVALUE: int = 2
STRING_ERROR_MISSING_BLOCK: str = f"""No metadata block found. The metadata block must follow this format:
@@ -92,12 +92,6 @@ class Tag_boolean(NamedTuple):
""")
-STRING_WARNING_NO_MATCH: Template = Template(f"""This metadata line did not match the expected pattern and was ignored:
-
- $line
-
-""")
-
def isWhitespaceLine(s: str) -> bool:
return isSomething(re.compile(r"^\s*$").match(s))
@@ -107,9 +101,9 @@ def isCommentLine(s: str) -> bool:
def extract(userscriptContent: str) -> str: # raises MetadataError
match_metadataBlock: Optional[Match] = REGEX_METADATA_BLOCK.search(userscriptContent)
- if (match_metadataBlock == None):
+ if (match_metadataBlock is None):
raise MetadataError(STRING_ERROR_MISSING_BLOCK)
- block: str = match_metadataBlock.group(INDEX_GROUP_BLOCK_CONTENT)
+ block: str = match_metadataBlock.group(REGEXGROUP_CONTENT)
for line in block.splitlines():
if not isWhitespaceLine(line) and not isCommentLine(line) and not REGEX_METADATA_LINE.match(line):
raise MetadataError(STRING_ERROR_INVALID_BLOCK.substitute(line=line))
@@ -119,18 +113,23 @@ def extract(userscriptContent: str) -> str: # raises MetadataError
def parse(metadataContent: str) -> Metadata:
def parseLine(line: str) -> Optional[MetadataItem]:
match: Optional[Match] = REGEX_METADATA_LINE.search(line)
- if match == None:
- # if not REGEX_EMPTY_LINE_COMMENT.match(line): # TODO: uncomment when we can handle warnings
- # warnings.warn(STRING_WARNING_NO_MATCH.substitute(line=line))
+ if match is None:
return None
else:
- tagName: str = match.group(INDEX_GROUP_TAGNAME)
- tagValue: Optional[str] = match.group(INDEX_GROUP_TAGVALUE)
- return (tagName, True if tagValue == None else tagValue) # Boolean metadata tags have no explicit value; if they are present, they are true.
-
- return list(filter(isSomething,
- map(parseLine, metadataContent.splitlines())
- ))
+ tagName: str = match.group(REGEXGROUP_TAGNAME)
+ tagValue: Optional[str] = match.group(REGEXGROUP_TAGVALUE)
+ # Boolean tags have no explicit value; if they are present, they are true:
+ if tagValue is None:
+ return (tagName, True)
+ else:
+ return (tagName, tagValue)
+
+ # filter did not play well with mypy:
+ parsedItems: Metadata = []
+ for item in map(parseLine, metadataContent.splitlines()):
+ if item is not None:
+ parsedItems.append(item)
+ return parsedItems
def tagByName(tags: List[Tag], tagName: str) -> Optional[Tag]:
@@ -139,8 +138,8 @@ def tagByName(tags: List[Tag], tagName: str) -> Optional[Tag]:
def validatePair(tags: List[Tag], pair: MetadataItem) -> MetadataItem:
(tagName, tagValue) = pair
- tag: Tag = tagByName(tags, tagName)
- if tag == None:
+ tag: Optional[Tag] = tagByName(tags, tagName)
+ if tag is None:
# Unrecognized key.
return (tagName, tagValue)
else:
@@ -148,64 +147,66 @@ def validatePair(tags: List[Tag], pair: MetadataItem) -> MetadataItem:
tagPredicate: Optional[Predicate] = tag.predicate
if type(tag) is Tag_string and type(tagValue) is not str:
raise MetadataError(STRING_ERROR_MISSING_VALUE.substitute(tagName=tagName))
- if type(tag) is Tag_boolean and type(tagValue) is not bool:
- tagValue = True # because a boolean directive which is present is true no matter what comes after it
- if isSomething(tagPredicate):
+ if type(tag) is Tag_boolean:
+ tagValue = tagValue is not False # This handles cases like `@noframes blabla`; a boolean directive is true no matter what comes after it.
+ if tagPredicate is not None:
if not tagPredicate(tagValue):
raise MetadataError(STRING_ERROR_PREDICATE_FAILED.substitute(tagName=tagName, tagValue=str(tagValue)))
return (tagName, tagValue)
def validate(tags: List[Tag], metadata: Metadata) -> Metadata: # raises MetadataError
- def handleDuplicate(acc: Metadata, pair: MetadataItem) -> Metadata:
- (name, val) = pair
- tag: Tag = tagByName(tags, name)
- if tag == None:
- # Unrecognized tag. Just let it pass.
- return acc + [pair]
- else:
- # Recognized tag! Skip it if it is a duplicate of a unique key.
- seenTagNames: Iterator[str] = map(first, acc)
- return acc if tag.unique and name in seenTagNames else acc + [pair]
+ def handleDuplicate(acc: Iterable[MetadataItem], pair: MetadataItem) -> Iterable[MetadataItem]:
+ name: str = first(pair)
+ tag: Optional[Tag] = tagByName(tags, name)
+ seenTagNames: Iterator[str] = map(first, acc)
+ # Throw away pair if it has the same tag name as some already seen, known, unique directive:
+ return acc if tag is not None and tag.unique and name in seenTagNames else list(acc) + [pair]
def withoutDuplicates(metadata: Metadata) -> Metadata:
- return reduce(handleDuplicate, metadata, [])
+ empty: Iterable[Tuple[str, Union[str, bool]]] = [] # to satisfy mypy
+ return list(reduce(handleDuplicate, metadata, empty))
+ # Awkwardly written to satisfy mypy:
def withDefaults(metadata: Metadata) -> Metadata:
tagNamesThatWeHave: List[str] = list(map(first, metadata))
- def hasDefaultAndNotAlreadyParsed(tag: Tag) -> bool:
- return isSomething(tag.default) and tag.name not in tagNamesThatWeHave
- neededDefaults: Metadata = list(map(
+ unseenItems = map(
lambda tag: (tag.name, tag.default),
filter(
- hasDefaultAndNotAlreadyParsed,
+ lambda tag: tag.name not in tagNamesThatWeHave,
tags
)
- ))
+ )
+ neededDefaults: Metadata = []
+ for (tagName, default) in unseenItems:
+ if default is not None:
+ neededDefaults.append((tagName, default))
return metadata + neededDefaults
def assertRequiredPresent(metadata: Metadata) -> Metadata:
ourTagNames: Iterator[str] = map(first, metadata)
- requiredTags: Iterator[Tag] = filter(lambda tag: tag.required== True, tags)
+ requiredTags: Iterator[Tag] = filter(lambda tag: tag.required, tags)
for tag in requiredTags:
if (tag.name not in ourTagNames):
raise MetadataError(STRING_ERROR_MISSING_TAG.substitute(tagName=tag.name))
return metadata
- return list(map(lambda *args: validatePair(tags, *args),
+ return list(map(
+ lambda *args: validatePair(tags, *args),
withDefaults(withoutDuplicates(
assertRequiredPresent(metadata)
))
))
-def validateWith(tags: List[Tag]):
+def validator(tags: List[Tag]) -> Callable[[Metadata], Metadata]:
return lambda metadata: validate(tags, metadata)
def valueGetter_all(metadata: Metadata) -> Callable[[Tag], List[TagValue]]:
return lambda tag: [second(pair) for pair in metadata if first(pair) == tag.name]
+
def valueGetter_one(metadata: Metadata) -> Callable[[Tag], Optional[TagValue]]:
v = valueGetter_all(metadata)
return lambda tag: None if len(v(tag)) == 0 else v(tag)[0]
diff --git a/modules/misc.py b/modules/misc.py
new file mode 100644
index 0000000..67ad9fe
--- /dev/null
+++ b/modules/misc.py
@@ -0,0 +1,3 @@
+# mitm[dump|proxy] cannot have hyphens in option keys.
+def sanitize(optionName: str) -> str:
+ return optionName.replace("-", "_")
diff --git a/modules/patterns.py b/modules/patterns.py
new file mode 100644
index 0000000..1c70be1
--- /dev/null
+++ b/modules/patterns.py
@@ -0,0 +1,81 @@
+from typing import Optional, Pattern, Match
+import re
+from modules.utilities import first, isSomething
+
+REGEX_MATCH_ALL = r""
+REGEX_MATCH_SCHEME = r"\*|https?"
+REGEXGROUP_MATCH_SCHEME = r"scheme"
+REGEX_MATCH_HOST = r"(\*\.)*[^\/\*]+|\*"
+REGEXGROUP_MATCH_HOST = r"host"
+REGEX_MATCH_PATH = r"\/.*"
+REGEXGROUP_MATCH_PATH = r"path"
+MATCH_PATTERN_ALL_NORMALIZED = "*://*/*"
+
+# Outer parentheses necessary to enclose `|`:
+REGEX_MATCH_PATTERN = re.compile(
+ r"^(?:" + REGEX_MATCH_ALL + r"|"
+ + r"(?P<" + REGEXGROUP_MATCH_SCHEME + r">" + REGEX_MATCH_SCHEME + r"):\/\/"
+ + r"(?P<" + REGEXGROUP_MATCH_HOST + r">" + REGEX_MATCH_HOST + r")"
+ + r"(?P<" + REGEXGROUP_MATCH_PATH + r">" + REGEX_MATCH_PATH + r")"
+ + r")$"
+)
+
+REGEX_INCLUDE_REGULAR = r"^(.+)$"
+REGEX_INCLUDE_REGEX = r"^\/(.+)\/$"
+
+REGEX_INCLUDE_PATTERN = re.compile(
+ REGEX_INCLUDE_REGEX + "|" + REGEX_INCLUDE_REGULAR
+)
+
+
+def normalizeMatchPattern(pattern: str) -> str:
+ return MATCH_PATTERN_ALL_NORMALIZED if pattern == REGEX_MATCH_ALL else pattern
+
+
+def isMatchPattern(pattern: str) -> bool:
+ return isSomething(REGEX_MATCH_PATTERN.match(pattern))
+
+
+def isIncludePattern(pattern: str) -> bool:
+ return isSomething(REGEX_INCLUDE_PATTERN.match(pattern))
+
+
+def isIncludePattern_regex(pattern: str) -> bool:
+ return isSomething(re.compile(REGEX_INCLUDE_REGEX).match(pattern))
+
+
+def withoutSurroundingSlashes(s: str) -> str:
+ return first(re.subn(re.compile(r"^\/|\/$"), "", s))
+
+
+def regexFromIncludePattern(pattern: str) -> Pattern: # raises re.error
+ return (
+ re.compile(withoutSurroundingSlashes(pattern), re.IGNORECASE)
+ if isIncludePattern_regex(pattern)
+ else re.compile(r"^" + regexify(pattern) + r"$", re.IGNORECASE)
+ )
+
+
+def regexify(segment: str) -> str:
+ return re.escape(segment).replace(r"\*", ".*")
+
+
+# Returns None if the pattern is invalid:
+def extractGroup(group: str, matchPattern: str) -> Optional[str]:
+ try:
+ match: Optional[Match] = REGEX_MATCH_PATTERN.search(normalizeMatchPattern(matchPattern))
+ return None if match is None else match.group(group)
+ except:
+ return None
+
+
+def schemeIn(matchPattern: str) -> Optional[str]:
+ return extractGroup(REGEXGROUP_MATCH_SCHEME, matchPattern)
+
+
+def hostIn(matchPattern: str) -> Optional[str]:
+ return extractGroup(REGEXGROUP_MATCH_HOST, matchPattern)
+
+
+def pathIn(matchPattern: str) -> Optional[str]:
+ return extractGroup(REGEXGROUP_MATCH_PATH, matchPattern)
diff --git a/modules/requests.py b/modules/requests.py
new file mode 100644
index 0000000..f988898
--- /dev/null
+++ b/modules/requests.py
@@ -0,0 +1,18 @@
+# Due to its mitmproxy dependency, this module can only be imported by files run
+# "in" mitm[dump|proxy], e.g. not in the launcher script.
+
+import re
+from typing import Callable, Optional, Pattern
+from mitmproxy import http
+from modules.utilities import equals
+
+CONTENT_TYPE: str = "Content-Type"
+REGEX_CHARSET: Pattern = re.compile(r"charset=([^;\s]+)")
+
+def inferEncoding(response: http.HTTPResponse) -> Optional[str]:
+ httpHeaderValue = response.headers.get(CONTENT_TYPE, "").lower()
+ match = REGEX_CHARSET.search(httpHeaderValue)
+ return match.group(1) if match else None
+
+def requestContainsQueryParam(param: str, request: http.HTTPRequest) -> bool:
+ return any(map(equals(param), request.query))
diff --git a/modules/text.py b/modules/text.py
new file mode 100644
index 0000000..0ca6b8d
--- /dev/null
+++ b/modules/text.py
@@ -0,0 +1,47 @@
+from modules.constants import DEFAULT_PORT, APP_NAME, VERSION, VERSION_PREFIX, DEFAULT_QUERY_PARAM_TO_DISABLE
+
+metavar_file = "FILE"
+metavar_dir = "DIR"
+metavar_param = "PARAM"
+matching = f"matching any of the rules specified in {metavar_file} (file name or glob pattern)"
+
+description = "Inject userscripts using mitmproxy."
+
+option_ignore = "ignore"
+help_ignore = "Intercept all traffic except from hosts " + matching
+
+option_intercept = "intercept"
+help_intercept = "Intercept only traffic from hosts " + matching
+
+option_inline_short = "i"
+option_inline = "inline"
+help_inline = "Always insert userscripts inline, never linked"
+
+option_port_short = "p"
+option_port = "port"
+help_port = f"""mitmproxy port (default: {DEFAULT_PORT})"""
+
+option_query_param_to_disable_short = "q"
+option_query_param_to_disable = "query-param-to-disable"
+help_query_param_to_disable = f"""Disable userscripts when the request URL contains a PARAM query parameter, for example "foo" to disable userscripts for http://example.com?foo (default: {DEFAULT_QUERY_PARAM_TO_DISABLE})"""
+
+option_recursive_short = "r"
+option_recursive = "recursive"
+help_recursive = f"""Recurse into directories when looking for userscripts"""
+
+option_transparent_short = "t"
+option_transparent = "transparent"
+help_transparent = "Transparent mode"
+
+option_userscripts_short = "u"
+option_userscripts = "userscripts"
+help_userscripts = f"Load userscripts from directory {metavar_dir}"
+
+option_list_injected_short = "l"
+option_list_injected = "list-injected"
+help_list_injected = "Insert an HTML comment with a list of injected userscripts"
+
+def stringifyVersion(version: str) -> str:
+ return VERSION_PREFIX + version
+
+INFO_MESSAGE: str = APP_NAME + " " + stringifyVersion(VERSION)
diff --git a/userscript.py b/modules/userscript.py
similarity index 63%
rename from userscript.py
rename to modules/userscript.py
index a9c5653..1f48e70 100644
--- a/userscript.py
+++ b/modules/userscript.py
@@ -1,28 +1,32 @@
from typing import Optional, Tuple, List, NamedTuple, Callable, Pattern
import re
-import metadata
import warnings
from string import Template
-from utilities import first, second, isSomething, strs, compose2
-from metadata import Metadata, TagValue, PREFIX_TAG, Tag, Tag_string, Tag_boolean
from urlmatch import urlmatch
-from patterns import isMatchPattern, isIncludePattern, regexFromIncludePattern
+import modules.metadata as metadata
+import modules.inline as inline
+from modules.utilities import first, second, isSomething, strs, compose2, stripIndentation
+from modules.metadata import Metadata, PREFIX_TAG, Tag, Tag_string, Tag_boolean
+from modules.patterns import isMatchPattern, isIncludePattern, regexFromIncludePattern
class UserscriptError(Exception):
def __init__(self,*args,**kwargs):
Exception.__init__(self,*args,**kwargs)
-directive_name: str = "name"
-directive_version: str = "version"
-directive_run_at: str = "run-at"
-directive_match: str = "match"
-directive_include: str = "include"
-directive_exclude: str = "exclude"
-directive_noframes: str = "noframes"
+REGEX_URL: Pattern = re.compile(r"^https?://")
-document_end: str = "document-end"
-document_start: str = "document-start"
-document_idle: str = "document-idle"
+directive_name : str = "name"
+directive_version : str = "version"
+directive_run_at : str = "run-at"
+directive_match : str = "match"
+directive_include : str = "include"
+directive_exclude : str = "exclude"
+directive_noframes : str = "noframes"
+directive_downloadURL : str = "downloadURL"
+
+document_end : str = "document-end"
+document_start : str = "document-start"
+document_idle : str = "document-idle"
tag_name: Tag_string = Tag_string(
name = directive_name,
@@ -66,6 +70,20 @@ def __init__(self,*args,**kwargs):
required = False,
predicate = None,
)
+tag_version: Tag_string = Tag_string(
+ name = directive_version,
+ unique = True,
+ default = None,
+ required = False,
+ predicate = None,
+)
+tag_downloadURL: Tag_string = Tag_string(
+ name = directive_downloadURL,
+ unique = True,
+ default = None,
+ required = False,
+ predicate = lambda val: isSomething(REGEX_URL.match(val)),
+)
METADATA_TAGS: List[Tag] = [
tag_name,
@@ -74,15 +92,12 @@ def __init__(self,*args,**kwargs):
tag_noframes,
tag_include,
tag_exclude,
- Tag_string(
- name = directive_version,
- unique = True,
- default = "0.0.0",
- required = False,
- predicate = None,
- ),
+ tag_version,
+ tag_downloadURL,
]
+validateMetadata: Callable[[Metadata], Metadata] = metadata.validator(METADATA_TAGS)
+
STRING_WARNING_INVALID_REGEX: Template = Template(f"""{PREFIX_TAG}{directive_include}/{PREFIX_TAG}{directive_exclude} patterns starting and ending with `/` are interpreted as regular expressions, and this pattern is not a valid regex:
@@ -97,19 +112,22 @@ def __init__(self,*args,**kwargs):
class Userscript(NamedTuple):
name: str
+ version: Optional[str]
content: str
runAt: str
noframes: bool
matchPatterns: List[str]
includePatternRegexes: List[Pattern]
excludePatternRegexes: List[Pattern]
+ downloadURL: Optional[str]
+ unsafeSequences: List[str] # in