From 417abb1f4e64156971ebdd815b81495d03fdf5fe Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 19 Nov 2017 04:04:04 +0100 Subject: [PATCH 001/103] Bump version number to 0.2.0 --- userscript-proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userscript-proxy.py b/userscript-proxy.py index 02cfcb0..3519c36 100644 --- a/userscript-proxy.py +++ b/userscript-proxy.py @@ -13,7 +13,7 @@ def stringifyVersion(version: str) -> str: return VERSION_PREFIX + version -VERSION: str = "0.1.0" +VERSION: str = "0.2.0" VERSION_PREFIX: str = "v" WELCOME_MESSAGE: str = "Userscript Proxy " + stringifyVersion(VERSION) DIRS_USERSCRIPTS: List[str] = ["userscripts"] From 56420f0239303b027f63fa7c91c9226e30f2b8cc Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 19 Nov 2017 15:57:37 +0100 Subject: [PATCH 002/103] Clarify regex group identification --- metadata.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/metadata.py b/metadata.py index b3e415a..349c4f5 100644 --- a/metadata.py +++ b/metadata.py @@ -39,21 +39,22 @@ class Tag_boolean(NamedTuple): BLOCK_START: str = "==UserScript==" BLOCK_END: str = "==/UserScript==" +REGEXGROUP_CONTENT: str = "content" REGEX_EMPTY_LINE_COMMENT: Pattern = re.compile(r"^(?:\/\/)?\s*$") REGEX_METADATA_BLOCK: Pattern = re.compile( PREFIX_COMMENT + r"\s*" + BLOCK_START + r"\n" - + r"(.*)" + + r"(?P<" + REGEXGROUP_CONTENT + r">.*)" + PREFIX_COMMENT + r"\s*" + BLOCK_END, re.DOTALL ) -INDEX_GROUP_BLOCK_CONTENT: int = 1 +REGEXGROUP_TAGNAME: str = "tagname" +REGEXGROUP_TAGVALUE: str = "tagvalue" REGEX_METADATA_LINE: Pattern = re.compile( r"^\s*" + PREFIX_COMMENT + r"\s*" + PREFIX_TAG - + r"([^\s]+)(?:\s+?(\S.*)?)?$" + + r"(?P<" + REGEXGROUP_TAGNAME + r">[^\s]+)" + + r"(?:\s+?(?P<" + REGEXGROUP_TAGVALUE + r">\S.*)?)?$" ) -INDEX_GROUP_TAGNAME: int = 1 -INDEX_GROUP_TAGVALUE: int = 2 STRING_ERROR_MISSING_BLOCK: str = f"""No metadata block found. The metadata block must follow this format: @@ -109,7 +110,7 @@ def extract(userscriptContent: str) -> str: # raises MetadataError match_metadataBlock: Optional[Match] = REGEX_METADATA_BLOCK.search(userscriptContent) if (match_metadataBlock == None): raise MetadataError(STRING_ERROR_MISSING_BLOCK) - block: str = match_metadataBlock.group(INDEX_GROUP_BLOCK_CONTENT) + block: str = match_metadataBlock.group(REGEXGROUP_CONTENT) for line in block.splitlines(): if not isWhitespaceLine(line) and not isCommentLine(line) and not REGEX_METADATA_LINE.match(line): raise MetadataError(STRING_ERROR_INVALID_BLOCK.substitute(line=line)) @@ -124,8 +125,8 @@ def parseLine(line: str) -> Optional[MetadataItem]: # warnings.warn(STRING_WARNING_NO_MATCH.substitute(line=line)) return None else: - tagName: str = match.group(INDEX_GROUP_TAGNAME) - tagValue: Optional[str] = match.group(INDEX_GROUP_TAGVALUE) + tagName: str = match.group(REGEXGROUP_TAGNAME) + tagValue: Optional[str] = match.group(REGEXGROUP_TAGVALUE) return (tagName, True if tagValue == None else tagValue) # Boolean metadata tags have no explicit value; if they are present, they are true. return list(filter(isSomething, From 443311897563df06d0666f5dd85d741096108c53 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 19 Nov 2017 19:52:15 +0100 Subject: [PATCH 003/103] Clean up code --- metadata.py | 43 ++++++++++++++++++++----------------------- userscript-proxy.py | 9 ++++----- userscript.py | 26 ++++++++++++++------------ 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/metadata.py b/metadata.py index 349c4f5..f72bc46 100644 --- a/metadata.py +++ b/metadata.py @@ -1,8 +1,9 @@ -from typing import TypeVar, Tuple, List, Iterator, Pattern, Match, Optional, Union, Callable, NamedTuple +from typing import TypeVar, Tuple, List, Iterator, Iterable, Pattern, Match, Optional, Union, Callable, NamedTuple import re from string import Template from functools import reduce -from utilities import A, B, first, second, isSomething +from utilities import first, second, isSomething +import itertools import warnings class MetadataError(Exception): @@ -127,9 +128,10 @@ def parseLine(line: str) -> Optional[MetadataItem]: else: tagName: str = match.group(REGEXGROUP_TAGNAME) tagValue: Optional[str] = match.group(REGEXGROUP_TAGVALUE) - return (tagName, True if tagValue == None else tagValue) # Boolean metadata tags have no explicit value; if they are present, they are true. + return (tagName, tagValue if isSomething(tagValue) else True) # Boolean tags have no explicit value; if they are present, they are true. - return list(filter(isSomething, + return list(filter( + isSomething, map(parseLine, metadataContent.splitlines()) )) @@ -149,8 +151,8 @@ def validatePair(tags: List[Tag], pair: MetadataItem) -> MetadataItem: tagPredicate: Optional[Predicate] = tag.predicate if type(tag) is Tag_string and type(tagValue) is not str: raise MetadataError(STRING_ERROR_MISSING_VALUE.substitute(tagName=tagName)) - if type(tag) is Tag_boolean and type(tagValue) is not bool: - tagValue = True # because a boolean directive which is present is true no matter what comes after it + if type(tag) is Tag_boolean: + tagValue = True # This handles cases like `@noframes blabla`; a boolean directive is true no matter what comes after it. if isSomething(tagPredicate): if not tagPredicate(tagValue): raise MetadataError(STRING_ERROR_PREDICATE_FAILED.substitute(tagName=tagName, tagValue=str(tagValue))) @@ -158,19 +160,15 @@ def validatePair(tags: List[Tag], pair: MetadataItem) -> MetadataItem: def validate(tags: List[Tag], metadata: Metadata) -> Metadata: # raises MetadataError - def handleDuplicate(acc: Metadata, pair: MetadataItem) -> Metadata: - (name, val) = pair + def handleDuplicate(acc: Iterable[MetadataItem], pair: MetadataItem) -> Iterable[MetadataItem]: + name: str = first(pair) tag: Tag = tagByName(tags, name) - if tag == None: - # Unrecognized tag. Just let it pass. - return acc + [pair] - else: - # Recognized tag! Skip it if it is a duplicate of a unique key. - seenTagNames: Iterator[str] = map(first, acc) - return acc if tag.unique and name in seenTagNames else acc + [pair] + seenTagNames: Iterator[str] = map(first, acc) + # Throw away pair if it has the same tag name as some already seen, known, unique directive: + return acc if isSomething(tag) and tag.unique and name in seenTagNames else list(acc) + [pair] def withoutDuplicates(metadata: Metadata) -> Metadata: - return reduce(handleDuplicate, metadata, []) + return list(reduce(handleDuplicate, metadata, [])) def withDefaults(metadata: Metadata) -> Metadata: tagNamesThatWeHave: List[str] = list(map(first, metadata)) @@ -178,35 +176,34 @@ def hasDefaultAndNotAlreadyParsed(tag: Tag) -> bool: return isSomething(tag.default) and tag.name not in tagNamesThatWeHave neededDefaults: Metadata = list(map( lambda tag: (tag.name, tag.default), - filter( - hasDefaultAndNotAlreadyParsed, - tags - ) + filter(hasDefaultAndNotAlreadyParsed, tags) )) return metadata + neededDefaults def assertRequiredPresent(metadata: Metadata) -> Metadata: ourTagNames: Iterator[str] = map(first, metadata) - requiredTags: Iterator[Tag] = filter(lambda tag: tag.required== True, tags) + requiredTags: Iterator[Tag] = filter(lambda tag: tag.required, tags) for tag in requiredTags: if (tag.name not in ourTagNames): raise MetadataError(STRING_ERROR_MISSING_TAG.substitute(tagName=tag.name)) return metadata - return list(map(lambda *args: validatePair(tags, *args), + return list(map( + lambda *args: validatePair(tags, *args), withDefaults(withoutDuplicates( assertRequiredPresent(metadata) )) )) -def validateWith(tags: List[Tag]): +def validator(tags: List[Tag]) -> Callable[[Metadata], Metadata]: return lambda metadata: validate(tags, metadata) def valueGetter_all(metadata: Metadata) -> Callable[[Tag], List[TagValue]]: return lambda tag: [second(pair) for pair in metadata if first(pair) == tag.name] + def valueGetter_one(metadata: Metadata) -> Callable[[Tag], Optional[TagValue]]: v = valueGetter_all(metadata) return lambda tag: None if len(v(tag)) == 0 else v(tag)[0] diff --git a/userscript-proxy.py b/userscript-proxy.py index 3519c36..9a3ce40 100644 --- a/userscript-proxy.py +++ b/userscript-proxy.py @@ -1,11 +1,10 @@ from typing import List, Callable, Pattern -import glob, os, fnmatch, re +import glob, os, re from bs4 import BeautifulSoup from mitmproxy import ctx, http from metadata import MetadataError import userscript from userscript import Userscript, UserscriptError, document_end, document_start, document_idle -from warnings import warn from utilities import first, second import shlex import warnings @@ -63,7 +62,7 @@ def __init__(self): for unsafe_filename in glob.glob(PATTERN_USERSCRIPT): filename = shlex.quote(unsafe_filename) - logInfo("Found " + filename + ".") + logInfo("Loading " + filename + " ...") try: content = open(filename).read() except PermissionError: @@ -98,7 +97,7 @@ def response(self, flow: http.HTTPFlow): isApplicable: Callable[[Userscript], bool] = userscript.applicableChecker(flow.request.url) for script in self.userscripts: if isApplicable(script): - logInfo("Injecting %s into %s ..." % (script.name, flow.request.url)) + logInfo(f"Injecting {script.name} into {flow.request.url} ...") tag = soup.new_tag("script") if script.runAt == document_start: tag.string = script.content @@ -106,7 +105,7 @@ def response(self, flow: http.HTTPFlow): elif script.runAt == document_idle: tag.string = userscript.wrapInEventListener("load", script.content) soup.head.append(tag) - elif script.runAt == document_end: + else: tag.string = script.content soup.body.append(tag) flow.response.content = str(soup).encode("utf8") diff --git a/userscript.py b/userscript.py index a9c5653..53d940c 100644 --- a/userscript.py +++ b/userscript.py @@ -4,7 +4,7 @@ import warnings from string import Template from utilities import first, second, isSomething, strs, compose2 -from metadata import Metadata, TagValue, PREFIX_TAG, Tag, Tag_string, Tag_boolean +from metadata import Metadata, PREFIX_TAG, Tag, Tag_string, Tag_boolean from urlmatch import urlmatch from patterns import isMatchPattern, isIncludePattern, regexFromIncludePattern @@ -12,17 +12,17 @@ class UserscriptError(Exception): def __init__(self,*args,**kwargs): Exception.__init__(self,*args,**kwargs) -directive_name: str = "name" -directive_version: str = "version" -directive_run_at: str = "run-at" -directive_match: str = "match" -directive_include: str = "include" -directive_exclude: str = "exclude" -directive_noframes: str = "noframes" +directive_name : str = "name" +directive_version : str = "version" +directive_run_at : str = "run-at" +directive_match : str = "match" +directive_include : str = "include" +directive_exclude : str = "exclude" +directive_noframes : str = "noframes" -document_end: str = "document-end" -document_start: str = "document-start" -document_idle: str = "document-idle" +document_end : str = "document-end" +document_start : str = "document-start" +document_idle : str = "document-idle" tag_name: Tag_string = Tag_string( name = directive_name, @@ -83,6 +83,8 @@ def __init__(self,*args,**kwargs): ), ] +validateMetadata: Callable[[Metadata], Metadata] = metadata.validator(METADATA_TAGS) + STRING_WARNING_INVALID_REGEX: Template = Template(f"""{PREFIX_TAG}{directive_include}/{PREFIX_TAG}{directive_exclude} patterns starting and ending with `/` are interpreted as regular expressions, and this pattern is not a valid regex: @@ -109,7 +111,7 @@ def __str__(self) -> str: def create(content: str) -> Userscript: - validMetadata: Metadata = metadata.validate(METADATA_TAGS, metadata.parse(metadata.extract(content))) + validMetadata: Metadata = validateMetadata(metadata.parse(metadata.extract(content))) valueOf = metadata.valueGetter_one(validMetadata) allValuesOf = metadata.valueGetter_all(validMetadata) includePatternRegexes: List[Pattern] = list(filter( From 86bae1bdc0797e6355d5aa4c91932126e736a0e2 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Tue, 26 Dec 2017 01:26:10 +0100 Subject: [PATCH 004/103] Add support for arbitrary character encodings Instead of just using UTF-8 invariantly, we now extract the charset from the Content-Type header and use that when encoding. --- userscript-proxy.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/userscript-proxy.py b/userscript-proxy.py index 9a3ce40..372deb7 100644 --- a/userscript-proxy.py +++ b/userscript-proxy.py @@ -1,4 +1,4 @@ -from typing import List, Callable, Pattern +from typing import Optional, List, Callable, Pattern, Match import glob, os, re from bs4 import BeautifulSoup from mitmproxy import ctx, http @@ -18,6 +18,8 @@ def stringifyVersion(version: str) -> str: DIRS_USERSCRIPTS: List[str] = ["userscripts"] PATTERN_USERSCRIPT: str = "*.user.js" RELEVANT_CONTENT_TYPES: List[str] = ["text/html"] +CHARSET_DEFAULT: str = "utf-8" +REGEX_CHARSET: Pattern = re.compile(r"charset=([^;\s]+)") REGEX_TEXT_HTML: Pattern = re.compile(r"text/html") TAB: str = " " @@ -92,7 +94,8 @@ def __init__(self): def response(self, flow: http.HTTPFlow): if "Content-Type" in flow.response.headers: - if REGEX_TEXT_HTML.match(flow.response.headers["Content-Type"]): + contentType: str = flow.response.headers["Content-Type"]; + if REGEX_TEXT_HTML.match(contentType): soup = BeautifulSoup(flow.response.content, "html.parser") # TODO: maybe change parser isApplicable: Callable[[Userscript], bool] = userscript.applicableChecker(flow.request.url) for script in self.userscripts: @@ -108,7 +111,10 @@ def response(self, flow: http.HTTPFlow): else: tag.string = script.content soup.body.append(tag) - flow.response.content = str(soup).encode("utf8") + # Keep character encoding: + match_charset: Optional[Match] = REGEX_CHARSET.search(contentType) + charset: str = CHARSET_DEFAULT if match_charset == None else match_charset.group(1) + flow.response.content = str(soup).encode(charset) def start(): From 15dde0b35ba45272fe11f2fd761de510fa5ec510 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Tue, 26 Dec 2017 19:05:23 +0100 Subject: [PATCH 005/103] Add named groups to match pattern regex --- patterns.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/patterns.py b/patterns.py index 312c82b..baf8289 100644 --- a/patterns.py +++ b/patterns.py @@ -4,12 +4,19 @@ REGEX_MATCH_ALL = r"" REGEX_MATCH_SCHEME = r"\*|https?" +REGEXGROUP_MATCH_SCHEME = r"scheme" REGEX_MATCH_HOST = r"(\*\.)*[^\/\*]+|\*" +REGEXGROUP_MATCH_HOST = r"host" REGEX_MATCH_PATH = r"\/.*" +REGEXGROUP_MATCH_PATH = r"path" # Outer parentheses necessary to enclose `|`: REGEX_MATCH_PATTERN = re.compile( - r"^(?:" + REGEX_MATCH_ALL + r"|(" + REGEX_MATCH_SCHEME + r"):\/\/(" + REGEX_MATCH_HOST + r")(" + REGEX_MATCH_PATH + r"))$" + r"^(?:" + REGEX_MATCH_ALL + r"|" + + r"(?P<" + REGEXGROUP_MATCH_SCHEME + r">" + REGEX_MATCH_SCHEME + r"):\/\/" + + r"(?P<" + REGEXGROUP_MATCH_HOST + r">" + REGEX_MATCH_HOST + r")" + + r"(?P<" + REGEXGROUP_MATCH_PATH + r">" + REGEX_MATCH_PATH + r")" + + r")$" ) REGEX_INCLUDE_REGULAR = r"^(.+)$" From 3fdd9110dc24970306be4bd22f5749aa11b71cff Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Tue, 26 Dec 2017 20:43:12 +0100 Subject: [PATCH 006/103] Add regex groups helper functions --- patterns.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/patterns.py b/patterns.py index baf8289..bfe5181 100644 --- a/patterns.py +++ b/patterns.py @@ -1,4 +1,4 @@ -from typing import Pattern +from typing import Optional, Pattern import re from utilities import first, isSomething @@ -9,6 +9,7 @@ REGEXGROUP_MATCH_HOST = r"host" REGEX_MATCH_PATH = r"\/.*" REGEXGROUP_MATCH_PATH = r"path" +MATCH_PATTERN_ALL_NORMALIZED = "*://*/*" # Outer parentheses necessary to enclose `|`: REGEX_MATCH_PATTERN = re.compile( @@ -26,6 +27,8 @@ REGEX_INCLUDE_REGEX + "|" + REGEX_INCLUDE_REGULAR ) +def normalizeMatchPattern(pattern: str) -> str: + return MATCH_PATTERN_ALL_NORMALIZED if pattern == REGEX_MATCH_ALL else pattern def isMatchPattern(pattern: str) -> bool: return isSomething(REGEX_MATCH_PATTERN.match(pattern)) @@ -43,5 +46,24 @@ def regexFromIncludePattern(pattern: str) -> Pattern: # raises re.error return ( re.compile(withoutSurroundingSlashes(pattern), re.IGNORECASE) if isIncludePattern_regex(pattern) - else re.compile(r"^" + re.escape(pattern).replace(r"\*", ".*") + r"$", re.IGNORECASE) + else re.compile(r"^" + regexify(pattern) + r"$", re.IGNORECASE) ) + +def regexify(segment: str) -> str: + return re.escape(segment).replace(r"\*", ".*") + +# Returns None if the pattern is invalid: +def extractGroup(group: str, matchPattern: str) -> Optional[str]: + try: + return REGEX_MATCH_PATTERN.search(normalizeMatchPattern(matchPattern)).group(group) + except: + return None + +def schemeIn(matchPattern: str) -> Optional[str]: + return extractGroup(REGEXGROUP_MATCH_SCHEME, matchPattern) + +def hostIn(matchPattern: str) -> Optional[str]: + return extractGroup(REGEXGROUP_MATCH_HOST, matchPattern) + +def pathIn(matchPattern: str) -> Optional[str]: + return extractGroup(REGEXGROUP_MATCH_PATH, matchPattern) From e5f2c18379d432c50a44a0fc60d44e35fde5920f Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Tue, 26 Dec 2017 20:44:07 +0100 Subject: [PATCH 007/103] Convert indentation to spaces --- patterns.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/patterns.py b/patterns.py index bfe5181..f42d165 100644 --- a/patterns.py +++ b/patterns.py @@ -13,41 +13,41 @@ # Outer parentheses necessary to enclose `|`: REGEX_MATCH_PATTERN = re.compile( - r"^(?:" + REGEX_MATCH_ALL + r"|" - + r"(?P<" + REGEXGROUP_MATCH_SCHEME + r">" + REGEX_MATCH_SCHEME + r"):\/\/" - + r"(?P<" + REGEXGROUP_MATCH_HOST + r">" + REGEX_MATCH_HOST + r")" - + r"(?P<" + REGEXGROUP_MATCH_PATH + r">" + REGEX_MATCH_PATH + r")" - + r")$" + r"^(?:" + REGEX_MATCH_ALL + r"|" + + r"(?P<" + REGEXGROUP_MATCH_SCHEME + r">" + REGEX_MATCH_SCHEME + r"):\/\/" + + r"(?P<" + REGEXGROUP_MATCH_HOST + r">" + REGEX_MATCH_HOST + r")" + + r"(?P<" + REGEXGROUP_MATCH_PATH + r">" + REGEX_MATCH_PATH + r")" + + r")$" ) REGEX_INCLUDE_REGULAR = r"^(.+)$" REGEX_INCLUDE_REGEX = r"^\/(.+)\/$" REGEX_INCLUDE_PATTERN = re.compile( - REGEX_INCLUDE_REGEX + "|" + REGEX_INCLUDE_REGULAR + REGEX_INCLUDE_REGEX + "|" + REGEX_INCLUDE_REGULAR ) def normalizeMatchPattern(pattern: str) -> str: return MATCH_PATTERN_ALL_NORMALIZED if pattern == REGEX_MATCH_ALL else pattern def isMatchPattern(pattern: str) -> bool: - return isSomething(REGEX_MATCH_PATTERN.match(pattern)) + return isSomething(REGEX_MATCH_PATTERN.match(pattern)) def isIncludePattern(pattern: str) -> bool: - return isSomething(REGEX_INCLUDE_PATTERN.match(pattern)) + return isSomething(REGEX_INCLUDE_PATTERN.match(pattern)) def isIncludePattern_regex(pattern: str) -> bool: - return isSomething(re.compile(REGEX_INCLUDE_REGEX).match(pattern)) + return isSomething(re.compile(REGEX_INCLUDE_REGEX).match(pattern)) def withoutSurroundingSlashes(s: str) -> str: - return first(re.subn(re.compile(r"^\/|\/$"), "", s)) + return first(re.subn(re.compile(r"^\/|\/$"), "", s)) def regexFromIncludePattern(pattern: str) -> Pattern: # raises re.error - return ( - re.compile(withoutSurroundingSlashes(pattern), re.IGNORECASE) + return ( + re.compile(withoutSurroundingSlashes(pattern), re.IGNORECASE) if isIncludePattern_regex(pattern) - else re.compile(r"^" + regexify(pattern) + r"$", re.IGNORECASE) - ) + else re.compile(r"^" + regexify(pattern) + r"$", re.IGNORECASE) + ) def regexify(segment: str) -> str: return re.escape(segment).replace(r"\*", ".*") From fac11c2ea5c7a56e88f913d37f5c1a0cf0b16794 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 27 Dec 2017 16:33:23 +0100 Subject: [PATCH 008/103] Add ignore module --- ignore.py | 32 ++++++++++++++++++++++++++++++++ utilities.py | 9 +++++++++ 2 files changed, 41 insertions(+) create mode 100644 ignore.py diff --git a/ignore.py b/ignore.py new file mode 100644 index 0000000..265bd96 --- /dev/null +++ b/ignore.py @@ -0,0 +1,32 @@ +from typing import List +import re +from utilities import compose2, not_, beginsWith +from patterns import isIncludePattern_regex, regexify, withoutSurroundingSlashes + +PREFIX_COMMENT: str = "#" +PIPE: str = "|" + +def rulesIn(text: str) -> List[str]: + return list(filter( + compose2(not_, beginsWith(PREFIX_COMMENT)), + filter(lambda s: s != "", text.splitlines()) + )) + + +def withPortSuffix(regex: str) -> str: + return regex + r"\:\d+" if re.compile(r":").search(regex) == None else regex + + +def ignoreRegex(ignoreRule: str) -> str: + return ( + withoutSurroundingSlashes(ignoreRule) + if isIncludePattern_regex(ignoreRule) + else r"^(?:.+\.)?" + withPortSuffix(regexify(ignoreRule)) + r"$" + ) + + +def entireIgnoreRegex(ignoreFileContent: str) -> str: + return PIPE.join(map( + ignoreRegex, + rulesIn(ignoreFileContent) + )) diff --git a/utilities.py b/utilities.py index 31362fa..a4e753a 100644 --- a/utilities.py +++ b/utilities.py @@ -4,6 +4,11 @@ B = TypeVar('B') C = TypeVar('C') + +def not_(x: bool) -> bool: + return not x + + def first(tuple: Tuple[A, B]) -> A: (a, b) = tuple return a @@ -24,3 +29,7 @@ def strs(xs: Any) -> List[str]: def compose2(f: Callable[[B], C], g: Callable[[A], B]) -> Callable[[A], C]: return lambda x: f(g(x)) + + +def beginsWith(prefix: str) -> Callable[[str], bool]: + return lambda s: s.startswith(prefix) From 1724fe82b75e4ab2ad35f6091e8f63c8a8cd37ff Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 27 Dec 2017 16:36:31 +0100 Subject: [PATCH 009/103] Two empty lines between defs --- patterns.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/patterns.py b/patterns.py index f42d165..970841d 100644 --- a/patterns.py +++ b/patterns.py @@ -27,21 +27,27 @@ REGEX_INCLUDE_REGEX + "|" + REGEX_INCLUDE_REGULAR ) + def normalizeMatchPattern(pattern: str) -> str: return MATCH_PATTERN_ALL_NORMALIZED if pattern == REGEX_MATCH_ALL else pattern + def isMatchPattern(pattern: str) -> bool: return isSomething(REGEX_MATCH_PATTERN.match(pattern)) + def isIncludePattern(pattern: str) -> bool: return isSomething(REGEX_INCLUDE_PATTERN.match(pattern)) + def isIncludePattern_regex(pattern: str) -> bool: return isSomething(re.compile(REGEX_INCLUDE_REGEX).match(pattern)) + def withoutSurroundingSlashes(s: str) -> str: return first(re.subn(re.compile(r"^\/|\/$"), "", s)) + def regexFromIncludePattern(pattern: str) -> Pattern: # raises re.error return ( re.compile(withoutSurroundingSlashes(pattern), re.IGNORECASE) @@ -49,9 +55,11 @@ def regexFromIncludePattern(pattern: str) -> Pattern: # raises re.error else re.compile(r"^" + regexify(pattern) + r"$", re.IGNORECASE) ) + def regexify(segment: str) -> str: return re.escape(segment).replace(r"\*", ".*") + # Returns None if the pattern is invalid: def extractGroup(group: str, matchPattern: str) -> Optional[str]: try: @@ -59,11 +67,14 @@ def extractGroup(group: str, matchPattern: str) -> Optional[str]: except: return None + def schemeIn(matchPattern: str) -> Optional[str]: return extractGroup(REGEXGROUP_MATCH_SCHEME, matchPattern) + def hostIn(matchPattern: str) -> Optional[str]: return extractGroup(REGEXGROUP_MATCH_HOST, matchPattern) + def pathIn(matchPattern: str) -> Optional[str]: return extractGroup(REGEXGROUP_MATCH_PATH, matchPattern) From dc6a5b6676b6380ed50d42464084118cc0858dd9 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 27 Dec 2017 17:45:56 +0100 Subject: [PATCH 010/103] Rework file structure As of this commit, UP is started like so: python3.6 userscript-proxy.py It then launches mitmdump with injector.py as a script. --- injector.py | 121 +++++++++++++++++++++++++++ ignore.py => lib/ignore.py | 4 +- metadata.py => lib/metadata.py | 2 +- patterns.py => lib/patterns.py | 2 +- userscript.py => lib/userscript.py | 8 +- utilities.py => lib/utilities.py | 0 userscript-proxy.py | 126 ++--------------------------- 7 files changed, 136 insertions(+), 127 deletions(-) create mode 100644 injector.py rename ignore.py => lib/ignore.py (84%) rename metadata.py => lib/metadata.py (99%) rename patterns.py => lib/patterns.py (98%) rename userscript.py => lib/userscript.py (94%) rename utilities.py => lib/utilities.py (100%) diff --git a/injector.py b/injector.py new file mode 100644 index 0000000..39503a8 --- /dev/null +++ b/injector.py @@ -0,0 +1,121 @@ +from typing import Optional, List, Callable, Pattern, Match +import glob, os, re +from bs4 import BeautifulSoup +from mitmproxy import ctx, http +import shlex +import warnings +from lib.metadata import MetadataError +import lib.userscript as userscript +from lib.userscript import Userscript, UserscriptError, document_end, document_start, document_idle +from lib.utilities import first, second + +def stringifyVersion(version: str) -> str: + return VERSION_PREFIX + version + +VERSION: str = "0.2.0" +VERSION_PREFIX: str = "v" +WELCOME_MESSAGE: str = "Userscript Proxy " + stringifyVersion(VERSION) +DIRS_USERSCRIPTS: List[str] = ["userscripts"] +PATTERN_USERSCRIPT: str = "*.user.js" +RELEVANT_CONTENT_TYPES: List[str] = ["text/html"] +CHARSET_DEFAULT: str = "utf-8" +REGEX_CHARSET: Pattern = re.compile(r"charset=([^;\s]+)") +REGEX_TEXT_HTML: Pattern = re.compile(r"text/html") +TAB: str = " " + +def logInfo(s: str) -> None: + try: + ctx.log.info(s) + except Exception: + print(s) + +def logWarning(s: str) -> None: + try: + ctx.log.warn(s) + except Exception: + print(s) + +def logError(s: str) -> None: + try: + ctx.log.error(s) + except Exception: + print(s) + +class UserscriptInjector: + def __init__(self): + self.userscripts: List[Userscript] = [] + logInfo("") + logInfo("╔═" + "═" * len(WELCOME_MESSAGE) + "═╗") + logInfo("║ " + WELCOME_MESSAGE + " ║") + logInfo("╚═" + "═" * len(WELCOME_MESSAGE) + "═╝") + logInfo("") + logInfo("Loading userscripts ...") + loadedUserscripts: List[Tuple[Userscript, str]] = [] + for directory in DIRS_USERSCRIPTS: + logInfo("Looking for userscripts (`"+PATTERN_USERSCRIPT+"`) in directory `"+directory+"` ...") + try: + os.chdir(directory) + except FileNotFoundError: + logWarning("Directory `"+directory+"` does not exist.") + continue + except PermissionError: + logError("Permission was denied when trying to read directory `"+DIR_USERSCRIPTS+"`.") + continue + + for unsafe_filename in glob.glob(PATTERN_USERSCRIPT): + filename = shlex.quote(unsafe_filename) + logInfo("Loading " + filename + " ...") + try: + content = open(filename).read() + except PermissionError: + logError("Could not read file `"+filename+"`: Permission denied.") + continue + except Exception as e: + logError("Could not read file `"+filename+"`: " + str(e)) + continue + try: + loadedUserscripts.append((userscript.create(content), filename)) + except MetadataError as err: + logError("Metadata error:") + logError(str(err)) + continue + except UserscriptError as err: + logError("Userscript error:") + logError(str(err)) + continue + + logInfo("") + logInfo(str(len(loadedUserscripts)) + " userscript(s) loaded:") + for s in loadedUserscripts: + logInfo(TAB + "• "+first(s).name+" ("+second(s)+")") + logInfo("") + self.userscripts = list(map(first, loadedUserscripts)) + + + def response(self, flow: http.HTTPFlow): + if "Content-Type" in flow.response.headers: + contentType: str = flow.response.headers["Content-Type"]; + if REGEX_TEXT_HTML.match(contentType): + soup = BeautifulSoup(flow.response.content, "html.parser") # TODO: maybe change parser + isApplicable: Callable[[Userscript], bool] = userscript.applicableChecker(flow.request.url) + for script in self.userscripts: + if isApplicable(script): + logInfo(f"Injecting {script.name} into {flow.request.url} ...") + tag = soup.new_tag("script") + if script.runAt == document_start: + tag.string = script.content + soup.head.append(tag) + elif script.runAt == document_idle: + tag.string = userscript.wrapInEventListener("load", script.content) + soup.head.append(tag) + else: + tag.string = script.content + soup.body.append(tag) + # Keep character encoding: + match_charset: Optional[Match] = REGEX_CHARSET.search(contentType) + charset: str = CHARSET_DEFAULT if match_charset == None else match_charset.group(1) + flow.response.content = str(soup).encode(charset) + + +def start(): + return UserscriptInjector() diff --git a/ignore.py b/lib/ignore.py similarity index 84% rename from ignore.py rename to lib/ignore.py index 265bd96..4d43c86 100644 --- a/ignore.py +++ b/lib/ignore.py @@ -1,7 +1,7 @@ from typing import List import re -from utilities import compose2, not_, beginsWith -from patterns import isIncludePattern_regex, regexify, withoutSurroundingSlashes +from lib.utilities import compose2, not_, beginsWith +from lib.patterns import isIncludePattern_regex, regexify, withoutSurroundingSlashes PREFIX_COMMENT: str = "#" PIPE: str = "|" diff --git a/metadata.py b/lib/metadata.py similarity index 99% rename from metadata.py rename to lib/metadata.py index f72bc46..be08788 100644 --- a/metadata.py +++ b/lib/metadata.py @@ -2,9 +2,9 @@ import re from string import Template from functools import reduce -from utilities import first, second, isSomething import itertools import warnings +from lib.utilities import first, second, isSomething class MetadataError(Exception): def __init__(self,*args,**kwargs): diff --git a/patterns.py b/lib/patterns.py similarity index 98% rename from patterns.py rename to lib/patterns.py index 970841d..1be121c 100644 --- a/patterns.py +++ b/lib/patterns.py @@ -1,6 +1,6 @@ from typing import Optional, Pattern import re -from utilities import first, isSomething +from lib.utilities import first, isSomething REGEX_MATCH_ALL = r"" REGEX_MATCH_SCHEME = r"\*|https?" diff --git a/userscript.py b/lib/userscript.py similarity index 94% rename from userscript.py rename to lib/userscript.py index 53d940c..afca19d 100644 --- a/userscript.py +++ b/lib/userscript.py @@ -1,12 +1,12 @@ from typing import Optional, Tuple, List, NamedTuple, Callable, Pattern import re -import metadata import warnings from string import Template -from utilities import first, second, isSomething, strs, compose2 -from metadata import Metadata, PREFIX_TAG, Tag, Tag_string, Tag_boolean from urlmatch import urlmatch -from patterns import isMatchPattern, isIncludePattern, regexFromIncludePattern +import lib.metadata as metadata +from lib.utilities import first, second, isSomething, strs, compose2 +from lib.metadata import Metadata, PREFIX_TAG, Tag, Tag_string, Tag_boolean +from lib.patterns import isMatchPattern, isIncludePattern, regexFromIncludePattern class UserscriptError(Exception): def __init__(self,*args,**kwargs): diff --git a/utilities.py b/lib/utilities.py similarity index 100% rename from utilities.py rename to lib/utilities.py diff --git a/userscript-proxy.py b/userscript-proxy.py index 372deb7..16c237c 100644 --- a/userscript-proxy.py +++ b/userscript-proxy.py @@ -1,121 +1,9 @@ -from typing import Optional, List, Callable, Pattern, Match -import glob, os, re -from bs4 import BeautifulSoup -from mitmproxy import ctx, http -from metadata import MetadataError -import userscript -from userscript import Userscript, UserscriptError, document_end, document_start, document_idle -from utilities import first, second -import shlex -import warnings +import subprocess -def stringifyVersion(version: str) -> str: - return VERSION_PREFIX + version +FILENAME_INJECTOR: str = "injector.py" -VERSION: str = "0.2.0" -VERSION_PREFIX: str = "v" -WELCOME_MESSAGE: str = "Userscript Proxy " + stringifyVersion(VERSION) -DIRS_USERSCRIPTS: List[str] = ["userscripts"] -PATTERN_USERSCRIPT: str = "*.user.js" -RELEVANT_CONTENT_TYPES: List[str] = ["text/html"] -CHARSET_DEFAULT: str = "utf-8" -REGEX_CHARSET: Pattern = re.compile(r"charset=([^;\s]+)") -REGEX_TEXT_HTML: Pattern = re.compile(r"text/html") -TAB: str = " " - -def logInfo(s: str) -> None: - try: - ctx.log.info(s) - except Exception: - print(s) - -def logWarning(s: str) -> None: - try: - ctx.log.warn(s) - except Exception: - print(s) - -def logError(s: str) -> None: - try: - ctx.log.error(s) - except Exception: - print(s) - -class UserscriptInjector: - def __init__(self): - self.userscripts: List[Userscript] = [] - logInfo("") - logInfo("╔═" + "═" * len(WELCOME_MESSAGE) + "═╗") - logInfo("║ " + WELCOME_MESSAGE + " ║") - logInfo("╚═" + "═" * len(WELCOME_MESSAGE) + "═╝") - logInfo("") - logInfo("Loading userscripts ...") - loadedUserscripts: List[Tuple[Userscript, str]] = [] - for directory in DIRS_USERSCRIPTS: - logInfo("Looking for userscripts (`"+PATTERN_USERSCRIPT+"`) in directory `"+directory+"` ...") - try: - os.chdir(directory) - except FileNotFoundError: - logWarning("Directory `"+directory+"` does not exist.") - continue - except PermissionError: - logError("Permission was denied when trying to read directory `"+DIR_USERSCRIPTS+"`.") - continue - - for unsafe_filename in glob.glob(PATTERN_USERSCRIPT): - filename = shlex.quote(unsafe_filename) - logInfo("Loading " + filename + " ...") - try: - content = open(filename).read() - except PermissionError: - logError("Could not read file `"+filename+"`: Permission denied.") - continue - except Exception as e: - logError("Could not read file `"+filename+"`: " + str(e)) - continue - try: - loadedUserscripts.append((userscript.create(content), filename)) - except MetadataError as err: - logError("Metadata error:") - logError(str(err)) - continue - except UserscriptError as err: - logError("Userscript error:") - logError(str(err)) - continue - - logInfo("") - logInfo(str(len(loadedUserscripts)) + " userscript(s) loaded:") - for s in loadedUserscripts: - logInfo(TAB + "• "+first(s).name+" ("+second(s)+")") - logInfo("") - self.userscripts = list(map(first, loadedUserscripts)) - - - def response(self, flow: http.HTTPFlow): - if "Content-Type" in flow.response.headers: - contentType: str = flow.response.headers["Content-Type"]; - if REGEX_TEXT_HTML.match(contentType): - soup = BeautifulSoup(flow.response.content, "html.parser") # TODO: maybe change parser - isApplicable: Callable[[Userscript], bool] = userscript.applicableChecker(flow.request.url) - for script in self.userscripts: - if isApplicable(script): - logInfo(f"Injecting {script.name} into {flow.request.url} ...") - tag = soup.new_tag("script") - if script.runAt == document_start: - tag.string = script.content - soup.head.append(tag) - elif script.runAt == document_idle: - tag.string = userscript.wrapInEventListener("load", script.content) - soup.head.append(tag) - else: - tag.string = script.content - soup.body.append(tag) - # Keep character encoding: - match_charset: Optional[Match] = REGEX_CHARSET.search(contentType) - charset: str = CHARSET_DEFAULT if match_charset == None else match_charset.group(1) - flow.response.content = str(soup).encode(charset) - - -def start(): - return UserscriptInjector() +try: + subprocess.run(["mitmdump", "-s", FILENAME_INJECTOR]) +except KeyboardInterrupt: + print("") + print("Interrupted by user.") From f35973377fa9fcb8044fb8f3980476297c7aef9f Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 27 Dec 2017 18:10:58 +0100 Subject: [PATCH 011/103] Minor code improvements --- injector.py | 5 +++-- lib/ignore.py | 7 ++++--- lib/metadata.py | 6 +++--- lib/utilities.py | 12 ++++++------ 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/injector.py b/injector.py index 39503a8..7286bdf 100644 --- a/injector.py +++ b/injector.py @@ -22,6 +22,7 @@ def stringifyVersion(version: str) -> str: REGEX_CHARSET: Pattern = re.compile(r"charset=([^;\s]+)") REGEX_TEXT_HTML: Pattern = re.compile(r"text/html") TAB: str = " " +HTML_PARSER: str = "html.parser" def logInfo(s: str) -> None: try: @@ -96,7 +97,7 @@ def response(self, flow: http.HTTPFlow): if "Content-Type" in flow.response.headers: contentType: str = flow.response.headers["Content-Type"]; if REGEX_TEXT_HTML.match(contentType): - soup = BeautifulSoup(flow.response.content, "html.parser") # TODO: maybe change parser + soup = BeautifulSoup(flow.response.content, HTML_PARSER) isApplicable: Callable[[Userscript], bool] = userscript.applicableChecker(flow.request.url) for script in self.userscripts: if isApplicable(script): @@ -113,7 +114,7 @@ def response(self, flow: http.HTTPFlow): soup.body.append(tag) # Keep character encoding: match_charset: Optional[Match] = REGEX_CHARSET.search(contentType) - charset: str = CHARSET_DEFAULT if match_charset == None else match_charset.group(1) + charset: str = CHARSET_DEFAULT if match_charset is None else match_charset.group(1) flow.response.content = str(soup).encode(charset) diff --git a/lib/ignore.py b/lib/ignore.py index 4d43c86..39ba4d1 100644 --- a/lib/ignore.py +++ b/lib/ignore.py @@ -3,18 +3,19 @@ from lib.utilities import compose2, not_, beginsWith from lib.patterns import isIncludePattern_regex, regexify, withoutSurroundingSlashes -PREFIX_COMMENT: str = "#" +COMMENT_PREFIX: str = "#" +PORT_PREFIX: str = ":" PIPE: str = "|" def rulesIn(text: str) -> List[str]: return list(filter( - compose2(not_, beginsWith(PREFIX_COMMENT)), + compose2(not_, beginsWith(COMMENT_PREFIX)), filter(lambda s: s != "", text.splitlines()) )) def withPortSuffix(regex: str) -> str: - return regex + r"\:\d+" if re.compile(r":").search(regex) == None else regex + return regex if PORT_PREFIX in regex else regex + r"\:\d+" def ignoreRegex(ignoreRule: str) -> str: diff --git a/lib/metadata.py b/lib/metadata.py index be08788..372856d 100644 --- a/lib/metadata.py +++ b/lib/metadata.py @@ -109,7 +109,7 @@ def isCommentLine(s: str) -> bool: def extract(userscriptContent: str) -> str: # raises MetadataError match_metadataBlock: Optional[Match] = REGEX_METADATA_BLOCK.search(userscriptContent) - if (match_metadataBlock == None): + if (match_metadataBlock is None): raise MetadataError(STRING_ERROR_MISSING_BLOCK) block: str = match_metadataBlock.group(REGEXGROUP_CONTENT) for line in block.splitlines(): @@ -121,7 +121,7 @@ def extract(userscriptContent: str) -> str: # raises MetadataError def parse(metadataContent: str) -> Metadata: def parseLine(line: str) -> Optional[MetadataItem]: match: Optional[Match] = REGEX_METADATA_LINE.search(line) - if match == None: + if match is None: # if not REGEX_EMPTY_LINE_COMMENT.match(line): # TODO: uncomment when we can handle warnings # warnings.warn(STRING_WARNING_NO_MATCH.substitute(line=line)) return None @@ -143,7 +143,7 @@ def tagByName(tags: List[Tag], tagName: str) -> Optional[Tag]: def validatePair(tags: List[Tag], pair: MetadataItem) -> MetadataItem: (tagName, tagValue) = pair tag: Tag = tagByName(tags, tagName) - if tag == None: + if tag is None: # Unrecognized key. return (tagName, tagValue) else: diff --git a/lib/utilities.py b/lib/utilities.py index a4e753a..0136201 100644 --- a/lib/utilities.py +++ b/lib/utilities.py @@ -1,4 +1,4 @@ -from typing import List, Callable, Any, TypeVar, Tuple +from typing import Optional, List, Callable, Any, TypeVar, Tuple A = TypeVar('A') B = TypeVar('B') @@ -6,7 +6,7 @@ def not_(x: bool) -> bool: - return not x + return not x def first(tuple: Tuple[A, B]) -> A: @@ -19,12 +19,12 @@ def second(tuple: Tuple[A, B]) -> B: return b -def isSomething(x: Any) -> bool: - return x != None +def isSomething(x: Optional[A]) -> bool: + return x is not None def strs(xs: Any) -> List[str]: - return list(map(str, xs)) + return list(map(str, xs)) def compose2(f: Callable[[B], C], g: Callable[[A], B]) -> Callable[[A], C]: @@ -32,4 +32,4 @@ def compose2(f: Callable[[B], C], g: Callable[[A], B]) -> Callable[[A], C]: def beginsWith(prefix: str) -> Callable[[str], bool]: - return lambda s: s.startswith(prefix) + return lambda s: s.startswith(prefix) From 30d0a6e566b3400bbd39d4ba254ee4dea4991bd3 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 27 Dec 2017 19:31:41 +0100 Subject: [PATCH 012/103] Add XHTML support --- injector.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/injector.py b/injector.py index 7286bdf..22dd487 100644 --- a/injector.py +++ b/injector.py @@ -17,10 +17,9 @@ def stringifyVersion(version: str) -> str: WELCOME_MESSAGE: str = "Userscript Proxy " + stringifyVersion(VERSION) DIRS_USERSCRIPTS: List[str] = ["userscripts"] PATTERN_USERSCRIPT: str = "*.user.js" -RELEVANT_CONTENT_TYPES: List[str] = ["text/html"] +RELEVANT_CONTENT_TYPES: List[str] = ["text/html", "application/xhtml+xml"] CHARSET_DEFAULT: str = "utf-8" REGEX_CHARSET: Pattern = re.compile(r"charset=([^;\s]+)") -REGEX_TEXT_HTML: Pattern = re.compile(r"text/html") TAB: str = " " HTML_PARSER: str = "html.parser" @@ -94,9 +93,11 @@ def __init__(self): def response(self, flow: http.HTTPFlow): - if "Content-Type" in flow.response.headers: - contentType: str = flow.response.headers["Content-Type"]; - if REGEX_TEXT_HTML.match(contentType): + HEADER_CONTENT_TYPE: str = "Content-Type" + if HEADER_CONTENT_TYPE in flow.response.headers: + contentType: str = flow.response.headers[HEADER_CONTENT_TYPE]; + if any(map(lambda t: t in contentType, RELEVANT_CONTENT_TYPES)): + # Response is a web page; proceed. soup = BeautifulSoup(flow.response.content, HTML_PARSER) isApplicable: Callable[[Userscript], bool] = userscript.applicableChecker(flow.request.url) for script in self.userscripts: From a60ddff9f71c3e2490c1423099713cab7208737b Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 27 Dec 2017 21:07:53 +0100 Subject: [PATCH 013/103] Inject info comment into HTML --- injector.py | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/injector.py b/injector.py index 22dd487..4b87413 100644 --- a/injector.py +++ b/injector.py @@ -1,6 +1,6 @@ -from typing import Optional, List, Callable, Pattern, Match +from typing import Optional, Iterable, List, Callable, Pattern, Match import glob, os, re -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup, Comment, Doctype from mitmproxy import ctx, http import shlex import warnings @@ -14,14 +14,19 @@ def stringifyVersion(version: str) -> str: VERSION: str = "0.2.0" VERSION_PREFIX: str = "v" -WELCOME_MESSAGE: str = "Userscript Proxy " + stringifyVersion(VERSION) +APP_NAME: str = "Userscript Proxy" +WELCOME_MESSAGE: str = APP_NAME + " " + stringifyVersion(VERSION) DIRS_USERSCRIPTS: List[str] = ["userscripts"] PATTERN_USERSCRIPT: str = "*.user.js" RELEVANT_CONTENT_TYPES: List[str] = ["text/html", "application/xhtml+xml"] CHARSET_DEFAULT: str = "utf-8" REGEX_CHARSET: Pattern = re.compile(r"charset=([^;\s]+)") TAB: str = " " +LIST_ITEM_PREFIX: str = TAB + "• " HTML_PARSER: str = "html.parser" +INFO_COMMENT_PREFIX: str = f""" +[{WELCOME_MESSAGE}] +""" def logInfo(s: str) -> None: try: @@ -41,6 +46,17 @@ def logError(s: str) -> None: except Exception: print(s) +def itemList(strs: Iterable[str]) -> str: + return "\n".join(map(lambda s: LIST_ITEM_PREFIX + s, strs)) + +def indexOfDoctype(soup: BeautifulSoup) -> Optional[int]: + index: int = 0 + for item in soup.contents: + if isinstance(item, Doctype): + return index + index += 1 + return None + class UserscriptInjector: def __init__(self): self.userscripts: List[Userscript] = [] @@ -86,8 +102,10 @@ def __init__(self): logInfo("") logInfo(str(len(loadedUserscripts)) + " userscript(s) loaded:") - for s in loadedUserscripts: - logInfo(TAB + "• "+first(s).name+" ("+second(s)+")") + logInfo(itemList(map( + lambda s: f"{first(s).name} ({second(s)})", + loadedUserscripts + ))) logInfo("") self.userscripts = list(map(first, loadedUserscripts)) @@ -98,11 +116,13 @@ def response(self, flow: http.HTTPFlow): contentType: str = flow.response.headers[HEADER_CONTENT_TYPE]; if any(map(lambda t: t in contentType, RELEVANT_CONTENT_TYPES)): # Response is a web page; proceed. + insertedScripts: List[str] = [] soup = BeautifulSoup(flow.response.content, HTML_PARSER) isApplicable: Callable[[Userscript], bool] = userscript.applicableChecker(flow.request.url) for script in self.userscripts: if isApplicable(script): logInfo(f"Injecting {script.name} into {flow.request.url} ...") + insertedScripts.append(script.name) tag = soup.new_tag("script") if script.runAt == document_start: tag.string = script.content @@ -113,6 +133,14 @@ def response(self, flow: http.HTTPFlow): else: tag.string = script.content soup.body.append(tag) + # Insert information comment: + index: Optional[int] = indexOfDoctype(soup) + soup.insert(0 if index is None else 1+index, Comment( + INFO_COMMENT_PREFIX + ( + "No matching userscripts for this URL." if insertedScripts == [] + else "These scripts were inserted:\n" + itemList(insertedScripts) + ) + "\n" + )) # Keep character encoding: match_charset: Optional[Match] = REGEX_CHARSET.search(contentType) charset: str = CHARSET_DEFAULT if match_charset is None else match_charset.group(1) From e6a8ccfd2a525ab576989d33102474cc676e3623 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Wed, 27 Dec 2017 23:58:23 +0100 Subject: [PATCH 014/103] Fix bug According to the HTML specification (see link below), the `DOCTYPE` part of the DTD is case-insensitive. So for example these DTDs are valid: But there is a bug in either BeautifulSoup or html.parser: If a document contains e.g. ``, then the invalid DTD `` is emitted. This commit adds a somewhat hacky fix for those cases. https://www.w3.org/TR/html51/syntax.html#the-doctype --- injector.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/injector.py b/injector.py index 4b87413..aa0abac 100644 --- a/injector.py +++ b/injector.py @@ -24,6 +24,7 @@ def stringifyVersion(version: str) -> str: TAB: str = " " LIST_ITEM_PREFIX: str = TAB + "• " HTML_PARSER: str = "html.parser" +REGEX_DOCTYPE: Pattern = re.compile(r"doctype\s+", re.I) INFO_COMMENT_PREFIX: str = f""" [{WELCOME_MESSAGE}] """ @@ -141,6 +142,10 @@ def response(self, flow: http.HTTPFlow): else "These scripts were inserted:\n" + itemList(insertedScripts) ) + "\n" )) + # Prevent BS/html.parser from emitting `` or similar if "DOCTYPE" is not all uppercase in source HTML: + if index is not None and REGEX_DOCTYPE.match(soup.contents[index]): + # There is a DTD and it is invalid, so replace it. + soup.contents[index] = Doctype(re.sub(REGEX_DOCTYPE, "", soup.contents[index])) # Keep character encoding: match_charset: Optional[Match] = REGEX_CHARSET.search(contentType) charset: str = CHARSET_DEFAULT if match_charset is None else match_charset.group(1) From 6baae11a8f1ff42f814c3026b5debe1b52aeaca1 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 28 Dec 2017 00:32:04 +0100 Subject: [PATCH 015/103] Implement ignore domains functionality --- ignore.txt | 38 ++++++++++++++++++++++++++++++++++++++ injector.py | 12 ++++++------ lib/ignore.py | 12 ++++++++---- lib/utilities.py | 10 +++------- userscript-proxy.py | 18 +++++++++++++++++- 5 files changed, 72 insertions(+), 18 deletions(-) create mode 100644 ignore.txt diff --git a/ignore.txt b/ignore.txt new file mode 100644 index 0000000..86555f5 --- /dev/null +++ b/ignore.txt @@ -0,0 +1,38 @@ +# Two kinds of rules: +# +# 1. @include pattern +# Asterisk ('*') means any string (including the empty string). '*.' is automatically prepended. ':*' is automatically appended unless the rule contains a colon (':'). +# To block a domain without blocking all its subdomains, use a regex rule (see below). +# +# 2. Regular expression +# If a rule starts and ends with a slash ('/'), it is treated as a (Python) regex. +# Note that the string to match against contains both a host and a port, e.g. 'example.com:443', and that the regex is used verbatim (i.e. you have to explicitly provide '^' etc if desired). The only exception is that the case-insensitivity flag ('?i') is automatically added. +# Be careful with '$': '/site.com$/' will never match, because it will only be used to check strings like 'site.com:80'. +# +# EXAMPLES: +# +# --- RULE ---------- MAKES MITMPROXY IGNORE ----------------------------------- +# site.com site.com and x.site.com +# api.site.com api.site.com and x.api.site.com, but not y.site.com +# *cdn.net cdn.net, fbcdn.net and x.fbcdn.net, but not cdn.net.com +# site.com:80 site.com:80 and x.site.com:80, but not site.com:443 +# /cdn\./ fbcdn.net, cdn.site.com, cdn.x.site.com, but not cdna.com +# /^site\.com:/ site.com, but not x.site.com, mysite.com or site.com.net +# +# Anything from a '#' until the end of the line is ignored, as well as leading and trailing whitespace. + +# Applications that cannot connect through mitmproxy: + itunes.apple.com # App Store + xp.apple.com # App Store + graph.facebook.com # Messenger + api.facebook.com # Messenger + edge-mqtt.facebook.com # Messenger + slack.com # Slack + +# Traffic irrelevant to Userscript Proxy: + /cdn\./ + i.ytimg.com # YouTube thumbnails + *.googlevideo.com # YouTube video content + api.twitch.tv # Twitch app metadata + ttvnw.net # Twitch video content + /^149\.154\.16[4-7]\.\d+:/ # Telegram Messenger Network diff --git a/injector.py b/injector.py index aa0abac..32496b1 100644 --- a/injector.py +++ b/injector.py @@ -2,12 +2,13 @@ import glob, os, re from bs4 import BeautifulSoup, Comment, Doctype from mitmproxy import ctx, http +from functools import partial import shlex import warnings from lib.metadata import MetadataError import lib.userscript as userscript from lib.userscript import Userscript, UserscriptError, document_end, document_start, document_idle -from lib.utilities import first, second +from lib.utilities import first, second, itemList def stringifyVersion(version: str) -> str: return VERSION_PREFIX + version @@ -47,9 +48,6 @@ def logError(s: str) -> None: except Exception: print(s) -def itemList(strs: Iterable[str]) -> str: - return "\n".join(map(lambda s: LIST_ITEM_PREFIX + s, strs)) - def indexOfDoctype(soup: BeautifulSoup) -> Optional[int]: index: int = 0 for item in soup.contents: @@ -58,6 +56,8 @@ def indexOfDoctype(soup: BeautifulSoup) -> Optional[int]: index += 1 return None +bulletList: Callable[[Iterable[str]], str] = partial(itemList, LIST_ITEM_PREFIX) + class UserscriptInjector: def __init__(self): self.userscripts: List[Userscript] = [] @@ -103,7 +103,7 @@ def __init__(self): logInfo("") logInfo(str(len(loadedUserscripts)) + " userscript(s) loaded:") - logInfo(itemList(map( + logInfo(bulletList(map( lambda s: f"{first(s).name} ({second(s)})", loadedUserscripts ))) @@ -139,7 +139,7 @@ def response(self, flow: http.HTTPFlow): soup.insert(0 if index is None else 1+index, Comment( INFO_COMMENT_PREFIX + ( "No matching userscripts for this URL." if insertedScripts == [] - else "These scripts were inserted:\n" + itemList(insertedScripts) + else "These scripts were inserted:\n" + bulletList(insertedScripts) ) + "\n" )) # Prevent BS/html.parser from emitting `` or similar if "DOCTYPE" is not all uppercase in source HTML: diff --git a/lib/ignore.py b/lib/ignore.py index 39ba4d1..cf252e0 100644 --- a/lib/ignore.py +++ b/lib/ignore.py @@ -1,19 +1,23 @@ -from typing import List +from typing import List, Pattern import re -from lib.utilities import compose2, not_, beginsWith from lib.patterns import isIncludePattern_regex, regexify, withoutSurroundingSlashes COMMENT_PREFIX: str = "#" PORT_PREFIX: str = ":" PIPE: str = "|" +REGEX_COMMENT: Pattern = re.compile(r"\#.*$") def rulesIn(text: str) -> List[str]: return list(filter( - compose2(not_, beginsWith(COMMENT_PREFIX)), - filter(lambda s: s != "", text.splitlines()) + lambda s: s != "", + map(withoutCommentAndTrimmed, text.splitlines()) )) +def withoutCommentAndTrimmed(line: str) -> str: + return re.sub(REGEX_COMMENT, "", line).strip() + + def withPortSuffix(regex: str) -> str: return regex if PORT_PREFIX in regex else regex + r"\:\d+" diff --git a/lib/utilities.py b/lib/utilities.py index 0136201..e2dd35f 100644 --- a/lib/utilities.py +++ b/lib/utilities.py @@ -1,14 +1,10 @@ -from typing import Optional, List, Callable, Any, TypeVar, Tuple +from typing import Optional, Iterable, List, Callable, Any, TypeVar, Tuple A = TypeVar('A') B = TypeVar('B') C = TypeVar('C') -def not_(x: bool) -> bool: - return not x - - def first(tuple: Tuple[A, B]) -> A: (a, b) = tuple return a @@ -31,5 +27,5 @@ def compose2(f: Callable[[B], C], g: Callable[[A], B]) -> Callable[[A], C]: return lambda x: f(g(x)) -def beginsWith(prefix: str) -> Callable[[str], bool]: - return lambda s: s.startswith(prefix) +def itemList(prefix: str, strs: Iterable[str]) -> str: + return "\n".join(map(lambda s: prefix + s, strs)) diff --git a/userscript-proxy.py b/userscript-proxy.py index 16c237c..da5f872 100644 --- a/userscript-proxy.py +++ b/userscript-proxy.py @@ -1,9 +1,25 @@ +from typing import List import subprocess +from lib.utilities import itemList +import lib.ignore as ignore FILENAME_INJECTOR: str = "injector.py" +FILENAME_IGNORE: str = "ignore.txt" try: - subprocess.run(["mitmdump", "-s", FILENAME_INJECTOR]) + print("Reading ignore rules ...") + ignoreFileContent: str = open(FILENAME_IGNORE).read() + rules: List[str] = ignore.rulesIn(ignoreFileContent) + print("Traffic from hosts matching any of these rules will be ignored by mitmproxy:") + print() + print(itemList(" ", rules)) + print() + regex: str = ignore.entireIgnoreRegex(ignoreFileContent) + subprocess.run(["mitmdump", "--ignore", regex, "-s", FILENAME_INJECTOR]) except KeyboardInterrupt: print("") print("Interrupted by user.") +except PermissionError: + print("Could not read file `"+FILENAME_IGNORE+"`: Permission denied.") +except Exception as e: + print("Could not read file `"+FILENAME_IGNORE+"`: " + str(e)) From e776cb9421e3eb0eca33102c5a690505adc2d6b9 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 28 Dec 2017 00:33:35 +0100 Subject: [PATCH 016/103] Bump version number to 0.3.0 --- injector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/injector.py b/injector.py index 32496b1..881d8c6 100644 --- a/injector.py +++ b/injector.py @@ -13,7 +13,7 @@ def stringifyVersion(version: str) -> str: return VERSION_PREFIX + version -VERSION: str = "0.2.0" +VERSION: str = "0.3.0" VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" WELCOME_MESSAGE: str = APP_NAME + " " + stringifyVersion(VERSION) From d8a52bcf8830c2895a4cd3ed78daf1430acd8997 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 28 Dec 2017 14:48:30 +0100 Subject: [PATCH 017/103] Show userscript version in info comment --- injector.py | 2 +- lib/userscript.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/injector.py b/injector.py index 881d8c6..d2454d6 100644 --- a/injector.py +++ b/injector.py @@ -123,7 +123,7 @@ def response(self, flow: http.HTTPFlow): for script in self.userscripts: if isApplicable(script): logInfo(f"Injecting {script.name} into {flow.request.url} ...") - insertedScripts.append(script.name) + insertedScripts.append(script.name + ("" if script.version is None else " " + stringifyVersion(script.version))) tag = soup.new_tag("script") if script.runAt == document_start: tag.string = script.content diff --git a/lib/userscript.py b/lib/userscript.py index afca19d..453179f 100644 --- a/lib/userscript.py +++ b/lib/userscript.py @@ -66,6 +66,13 @@ def __init__(self,*args,**kwargs): required = False, predicate = None, ) +tag_version: Tag_string = Tag_string( + name = directive_version, + unique = True, + default = None, + required = False, + predicate = None, +) METADATA_TAGS: List[Tag] = [ tag_name, @@ -74,13 +81,7 @@ def __init__(self,*args,**kwargs): tag_noframes, tag_include, tag_exclude, - Tag_string( - name = directive_version, - unique = True, - default = "0.0.0", - required = False, - predicate = None, - ), + tag_version, ] validateMetadata: Callable[[Metadata], Metadata] = metadata.validator(METADATA_TAGS) @@ -99,6 +100,7 @@ def __init__(self,*args,**kwargs): class Userscript(NamedTuple): name: str + version: Optional[str] content: str runAt: str noframes: bool @@ -130,6 +132,7 @@ def create(content: str) -> Userscript: )) return Userscript( name = str(valueOf(tag_name)), + version = None if valueOf(tag_version) is None else str(valueOf(tag_version)), content = content, runAt = str(valueOf(tag_run_at)), noframes = bool(valueOf(tag_noframes)), From 3ac8c1ffdf6953fbc87d7e0dc724a9e241a597f5 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 28 Dec 2017 15:05:41 +0100 Subject: [PATCH 018/103] Add UP version attribute to injected script tags --- injector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/injector.py b/injector.py index d2454d6..9a92873 100644 --- a/injector.py +++ b/injector.py @@ -26,6 +26,7 @@ def stringifyVersion(version: str) -> str: LIST_ITEM_PREFIX: str = TAB + "• " HTML_PARSER: str = "html.parser" REGEX_DOCTYPE: Pattern = re.compile(r"doctype\s+", re.I) +ATTRIBUTE_UP_VERSION: str = "data-userscript-proxy-version" INFO_COMMENT_PREFIX: str = f""" [{WELCOME_MESSAGE}] """ @@ -125,6 +126,7 @@ def response(self, flow: http.HTTPFlow): logInfo(f"Injecting {script.name} into {flow.request.url} ...") insertedScripts.append(script.name + ("" if script.version is None else " " + stringifyVersion(script.version))) tag = soup.new_tag("script") + tag[ATTRIBUTE_UP_VERSION] = VERSION if script.runAt == document_start: tag.string = script.content soup.head.append(tag) From 2cc9672cfd54ee0b7cf015da3e6a4d66cbdbeb72 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 28 Dec 2017 15:08:01 +0100 Subject: [PATCH 019/103] Wrap userscript content in line breaks --- injector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/injector.py b/injector.py index 9a92873..38cef7b 100644 --- a/injector.py +++ b/injector.py @@ -92,7 +92,7 @@ def __init__(self): logError("Could not read file `"+filename+"`: " + str(e)) continue try: - loadedUserscripts.append((userscript.create(content), filename)) + loadedUserscripts.append((userscript.create("\n" + content + "\n"), filename)) except MetadataError as err: logError("Metadata error:") logError(str(err)) From d987cc9f67f31953b79d12cad3e4e2085f0955f6 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 28 Dec 2017 16:21:51 +0100 Subject: [PATCH 020/103] Bump version number to 0.3.1 --- injector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/injector.py b/injector.py index 38cef7b..6a4157f 100644 --- a/injector.py +++ b/injector.py @@ -13,7 +13,7 @@ def stringifyVersion(version: str) -> str: return VERSION_PREFIX + version -VERSION: str = "0.3.0" +VERSION: str = "0.3.1" VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" WELCOME_MESSAGE: str = APP_NAME + " " + stringifyVersion(VERSION) From 2a72f5a0d5fcd220945cfdc0dc90becdd9724c0f Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 28 Dec 2017 16:31:38 +0100 Subject: [PATCH 021/103] Improve DTD related naming --- injector.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/injector.py b/injector.py index 6a4157f..1632f50 100644 --- a/injector.py +++ b/injector.py @@ -49,7 +49,7 @@ def logError(s: str) -> None: except Exception: print(s) -def indexOfDoctype(soup: BeautifulSoup) -> Optional[int]: +def indexOfDTD(soup: BeautifulSoup) -> Optional[int]: index: int = 0 for item in soup.contents: if isinstance(item, Doctype): @@ -137,17 +137,17 @@ def response(self, flow: http.HTTPFlow): tag.string = script.content soup.body.append(tag) # Insert information comment: - index: Optional[int] = indexOfDoctype(soup) - soup.insert(0 if index is None else 1+index, Comment( + index_DTD: Optional[int] = indexOfDTD(soup) + soup.insert(0 if index_DTD is None else 1+index_DTD, Comment( INFO_COMMENT_PREFIX + ( "No matching userscripts for this URL." if insertedScripts == [] else "These scripts were inserted:\n" + bulletList(insertedScripts) ) + "\n" )) # Prevent BS/html.parser from emitting `` or similar if "DOCTYPE" is not all uppercase in source HTML: - if index is not None and REGEX_DOCTYPE.match(soup.contents[index]): + if index_DTD is not None and REGEX_DOCTYPE.match(soup.contents[index_DTD]): # There is a DTD and it is invalid, so replace it. - soup.contents[index] = Doctype(re.sub(REGEX_DOCTYPE, "", soup.contents[index])) + soup.contents[index_DTD] = Doctype(re.sub(REGEX_DOCTYPE, "", soup.contents[index_DTD])) # Keep character encoding: match_charset: Optional[Match] = REGEX_CHARSET.search(contentType) charset: str = CHARSET_DEFAULT if match_charset is None else match_charset.group(1) From fb46721d5b4d517b45b72f93f0fb8f713ddac11c Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 28 Dec 2017 22:52:25 +0100 Subject: [PATCH 022/103] Fix metadata boolean directive bug Before this commit, boolean directives would erroneously evaluate to true even if they were not present, due to a logic bug. E.g. @noframes was always true. --- lib/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/metadata.py b/lib/metadata.py index 372856d..3188f70 100644 --- a/lib/metadata.py +++ b/lib/metadata.py @@ -152,7 +152,7 @@ def validatePair(tags: List[Tag], pair: MetadataItem) -> MetadataItem: if type(tag) is Tag_string and type(tagValue) is not str: raise MetadataError(STRING_ERROR_MISSING_VALUE.substitute(tagName=tagName)) if type(tag) is Tag_boolean: - tagValue = True # This handles cases like `@noframes blabla`; a boolean directive is true no matter what comes after it. + tagValue = tagValue is not False # This handles cases like `@noframes blabla`; a boolean directive is true no matter what comes after it. if isSomething(tagPredicate): if not tagPredicate(tagValue): raise MetadataError(STRING_ERROR_PREDICATE_FAILED.substitute(tagName=tagName, tagValue=str(tagValue))) From aa2027f2a3bc5d433e8b6d0ef108d6bdf998e313 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Thu, 28 Dec 2017 23:01:52 +0100 Subject: [PATCH 023/103] Implement @noframes --- injector.py | 13 +++++++++---- lib/userscript.py | 8 ++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/injector.py b/injector.py index 1632f50..e573d74 100644 --- a/injector.py +++ b/injector.py @@ -92,7 +92,7 @@ def __init__(self): logError("Could not read file `"+filename+"`: " + str(e)) continue try: - loadedUserscripts.append((userscript.create("\n" + content + "\n"), filename)) + loadedUserscripts.append((userscript.create(content), filename)) except MetadataError as err: logError("Metadata error:") logError(str(err)) @@ -127,14 +127,19 @@ def response(self, flow: http.HTTPFlow): insertedScripts.append(script.name + ("" if script.version is None else " " + stringifyVersion(script.version))) tag = soup.new_tag("script") tag[ATTRIBUTE_UP_VERSION] = VERSION + scriptContent: str = ( + "\n" + + (userscript.withNoframes(script.content) if script.noframes else script.content) + + "\n" + ) if script.runAt == document_start: - tag.string = script.content + tag.string = scriptContent soup.head.append(tag) elif script.runAt == document_idle: - tag.string = userscript.wrapInEventListener("load", script.content) + tag.string = userscript.wrapInEventListener("load", scriptContent) soup.head.append(tag) else: - tag.string = script.content + tag.string = scriptContent soup.body.append(tag) # Insert information comment: index_DTD: Optional[int] = indexOfDTD(soup) diff --git a/lib/userscript.py b/lib/userscript.py index 453179f..4e48bb9 100644 --- a/lib/userscript.py +++ b/lib/userscript.py @@ -161,6 +161,14 @@ def wrapInEventListener(event: str, scriptContent: str) -> str: return f"""window.addEventListener("{event}", function() {{\n{scriptContent}\n}});""" +def withNoframes(scriptContent: str) -> str: + return f"""if (window.top === window) {{ // {PREFIX_TAG + directive_noframes} + +{scriptContent} + +}}""" + + def regexFromIncludePattern_safe(pattern: str) -> Optional[Pattern]: try: return regexFromIncludePattern(pattern) From a7907860309f686e139f7f8adaf0546f1af93ff2 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Fri, 29 Dec 2017 00:21:15 +0100 Subject: [PATCH 024/103] Add basic readme --- README.md | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4c11792..efced2c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,37 @@ -# userscript-proxy -MITM userscript injector +# Userscript Proxy + +Browser extensions on iOS, Android and pretty much any other web browsing device. No jailbreak/root required. + +Userscript Proxy is built around [mitmproxy](mitmproxy) and acts as a MITM, injecting matching userscripts into web pages as they flow through it. Both HTTP and HTTPS are supported. + + +## Security + +**UP can (and must be able to) read all HTTP(S) traffic** sent to and from the device in question, so the only reasonably secure way to use it is to run it on a server controlled by oneself. + +Exceptions can be specified by adding ignore rules to `ignore.txt`. (This is even necessary for apps like Facebook Messenger and App Store, which refuse to connect through a MITM proxy.) **Traffic from/to hosts matched by such rules is not decrypted and cannot be read by mitmproxy or Userscript Proxy.** + + +## Data usage + +UP has negligible data usage impact when no userscript is injected, i.e. for URLs without any matching userscript. However, userscripts are injected into _every_ response from a matching URL, and the size of a userscript can be anything from a few hundred bytes for the most basic ones to hundreds of kilobytes in extreme cases. + +Many useful userscripts are relatively large, depending on everything from the functionality of the script to frameworks and compilation options. It is probably a good idea to have a general awareness of this issue, and to take appropriate action such as [minifying](minification) userscripts and adding suitable ignore rules. + + +## Functionality + +UP supports (a subset of) the [Greasemonkey metadata syntax](metadata). No adaptation of userscripts should be required. These directives are supported: + + * `@name` + * `@version` + * `@run-at document_(start|end|idle)` + * `@match` + * `@include` (basic pattern and regex) + * `@exclude` + * `@noframes` + + +[mitmproxy]: https://mitmproxy.org +[minification]: https://en.wikipedia.org/wiki/Minification_(programming) +[metadata]: https://wiki.greasespot.net/Metadata_Block From 4dc4cce36672b1d343a13f7ace6772317e9b2474 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Fri, 29 Dec 2017 00:22:41 +0100 Subject: [PATCH 025/103] Improve HTML emission and encoding As of this commit, prettify() is used, creating better looking HTML. It is now also explicitly defined that non-encodable characters should be replaced with '?' when encoding. --- injector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/injector.py b/injector.py index e573d74..49a3ba4 100644 --- a/injector.py +++ b/injector.py @@ -156,7 +156,7 @@ def response(self, flow: http.HTTPFlow): # Keep character encoding: match_charset: Optional[Match] = REGEX_CHARSET.search(contentType) charset: str = CHARSET_DEFAULT if match_charset is None else match_charset.group(1) - flow.response.content = str(soup).encode(charset) + flow.response.content = soup.prettify().encode(charset, "replace") def start(): From 1ffe2f13d40c5e79026c9ff9e900a2e6514bcdd6 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Fri, 29 Dec 2017 01:02:03 +0100 Subject: [PATCH 026/103] Infer and use charset correctly This is probably how decoding and encoding should be done. --- injector.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/injector.py b/injector.py index 49a3ba4..540532c 100644 --- a/injector.py +++ b/injector.py @@ -19,6 +19,7 @@ def stringifyVersion(version: str) -> str: WELCOME_MESSAGE: str = APP_NAME + " " + stringifyVersion(VERSION) DIRS_USERSCRIPTS: List[str] = ["userscripts"] PATTERN_USERSCRIPT: str = "*.user.js" +CONTENT_TYPE: str = "Content-Type" RELEVANT_CONTENT_TYPES: List[str] = ["text/html", "application/xhtml+xml"] CHARSET_DEFAULT: str = "utf-8" REGEX_CHARSET: Pattern = re.compile(r"charset=([^;\s]+)") @@ -59,6 +60,11 @@ def indexOfDTD(soup: BeautifulSoup) -> Optional[int]: bulletList: Callable[[Iterable[str]], str] = partial(itemList, LIST_ITEM_PREFIX) +def inferEncoding(response: http.HTTPResponse) -> Optional[str]: + httpHeaderValue = response.headers.get(CONTENT_TYPE, "").lower() + match = REGEX_CHARSET.search(httpHeaderValue) + return match.group(0) if match else None + class UserscriptInjector: def __init__(self): self.userscripts: List[Userscript] = [] @@ -113,13 +119,16 @@ def __init__(self): def response(self, flow: http.HTTPFlow): - HEADER_CONTENT_TYPE: str = "Content-Type" - if HEADER_CONTENT_TYPE in flow.response.headers: - contentType: str = flow.response.headers[HEADER_CONTENT_TYPE]; - if any(map(lambda t: t in contentType, RELEVANT_CONTENT_TYPES)): + response = flow.response + if CONTENT_TYPE in response.headers: + if any(map(lambda t: t in response.headers[CONTENT_TYPE], RELEVANT_CONTENT_TYPES)): # Response is a web page; proceed. insertedScripts: List[str] = [] - soup = BeautifulSoup(flow.response.content, HTML_PARSER) + soup = BeautifulSoup( + response.content, + HTML_PARSER, + from_encoding=inferEncoding(response) + ) isApplicable: Callable[[Userscript], bool] = userscript.applicableChecker(flow.request.url) for script in self.userscripts: if isApplicable(script): @@ -153,10 +162,11 @@ def response(self, flow: http.HTTPFlow): if index_DTD is not None and REGEX_DOCTYPE.match(soup.contents[index_DTD]): # There is a DTD and it is invalid, so replace it. soup.contents[index_DTD] = Doctype(re.sub(REGEX_DOCTYPE, "", soup.contents[index_DTD])) - # Keep character encoding: - match_charset: Optional[Match] = REGEX_CHARSET.search(contentType) - charset: str = CHARSET_DEFAULT if match_charset is None else match_charset.group(1) - flow.response.content = soup.prettify().encode(charset, "replace") + # Serialize and encode: + response.content = soup.prettify().encode( + soup.original_encoding if soup.original_encoding is not None else CHARSET_DEFAULT, + "replace" + ) def start(): From f254a8fe3ecb3a4a07b1f245e00279fce63810dd Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Fri, 29 Dec 2017 01:13:01 +0100 Subject: [PATCH 027/103] Improve injection reliability We should now be able to handle most cases of weirdly formatted HTML. --- injector.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/injector.py b/injector.py index 540532c..ba9f77d 100644 --- a/injector.py +++ b/injector.py @@ -141,15 +141,23 @@ def response(self, flow: http.HTTPFlow): (userscript.withNoframes(script.content) if script.noframes else script.content) + "\n" ) - if script.runAt == document_start: - tag.string = scriptContent - soup.head.append(tag) - elif script.runAt == document_idle: - tag.string = userscript.wrapInEventListener("load", scriptContent) - soup.head.append(tag) - else: - tag.string = scriptContent - soup.body.append(tag) + try: + if script.runAt == document_end: + tag.string = scriptContent + (soup.body if soup.body is not None else soup).append(tag) + else: + tag.string = scriptContent if script.runAt == document_start else userscript.wrapInEventListener("load", scriptContent) + if soup.head is not None: + soup.head.append(tag) + elif soup.title is not None: + soup.title.insert_after(tag) + elif soup.find() is not None: + soup.find().insert_before(tag) # before first element + else: + soup.append(tag) + except Exception as e: + logError("Injection failed due to the following error:") + logError(str(e)) # Insert information comment: index_DTD: Optional[int] = indexOfDTD(soup) soup.insert(0 if index_DTD is None else 1+index_DTD, Comment( From d0ff7b7957f657ac768efb3c10958e4645b27733 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Fri, 29 Dec 2017 02:03:36 +0100 Subject: [PATCH 028/103] Add fromOptional --- injector.py | 6 +++--- lib/utilities.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/injector.py b/injector.py index ba9f77d..1b4f611 100644 --- a/injector.py +++ b/injector.py @@ -8,7 +8,7 @@ from lib.metadata import MetadataError import lib.userscript as userscript from lib.userscript import Userscript, UserscriptError, document_end, document_start, document_idle -from lib.utilities import first, second, itemList +from lib.utilities import first, second, itemList, fromOptional def stringifyVersion(version: str) -> str: return VERSION_PREFIX + version @@ -144,7 +144,7 @@ def response(self, flow: http.HTTPFlow): try: if script.runAt == document_end: tag.string = scriptContent - (soup.body if soup.body is not None else soup).append(tag) + fromOptional(soup.body, soup).append(tag) else: tag.string = scriptContent if script.runAt == document_start else userscript.wrapInEventListener("load", scriptContent) if soup.head is not None: @@ -172,7 +172,7 @@ def response(self, flow: http.HTTPFlow): soup.contents[index_DTD] = Doctype(re.sub(REGEX_DOCTYPE, "", soup.contents[index_DTD])) # Serialize and encode: response.content = soup.prettify().encode( - soup.original_encoding if soup.original_encoding is not None else CHARSET_DEFAULT, + fromOptional(soup.original_encoding, CHARSET_DEFAULT), "replace" ) diff --git a/lib/utilities.py b/lib/utilities.py index e2dd35f..129f90c 100644 --- a/lib/utilities.py +++ b/lib/utilities.py @@ -27,5 +27,9 @@ def compose2(f: Callable[[B], C], g: Callable[[A], B]) -> Callable[[A], C]: return lambda x: f(g(x)) +def fromOptional(value: Optional[A], fallback: A) -> A: + return value if value is not None else fallback + + def itemList(prefix: str, strs: Iterable[str]) -> str: return "\n".join(map(lambda s: prefix + s, strs)) From a1f230fadf8ac8dbf2ece601a60c7883868a9f81 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Fri, 29 Dec 2017 02:35:26 +0100 Subject: [PATCH 029/103] Bump version number to 0.4.0 --- injector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/injector.py b/injector.py index 1b4f611..0b95680 100644 --- a/injector.py +++ b/injector.py @@ -13,7 +13,7 @@ def stringifyVersion(version: str) -> str: return VERSION_PREFIX + version -VERSION: str = "0.3.1" +VERSION: str = "0.4.0" VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" WELCOME_MESSAGE: str = APP_NAME + " " + stringifyVersion(VERSION) From 39e011fa64432d52cbdb6a8e6b03c3980be491a0 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Fri, 29 Dec 2017 02:49:12 +0100 Subject: [PATCH 030/103] Clarify security section in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index efced2c..91ed954 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ Userscript Proxy is built around [mitmproxy](mitmproxy) and acts as a MITM, inje ## Security -**UP can (and must be able to) read all HTTP(S) traffic** sent to and from the device in question, so the only reasonably secure way to use it is to run it on a server controlled by oneself. +**UP can (and must be able to) read and modify all HTTP(S) traffic** sent to and from the device in question, so the only reasonably secure way to use it is to run it on a server controlled by oneself. -Exceptions can be specified by adding ignore rules to `ignore.txt`. (This is even necessary for apps like Facebook Messenger and App Store, which refuse to connect through a MITM proxy.) **Traffic from/to hosts matched by such rules is not decrypted and cannot be read by mitmproxy or Userscript Proxy.** +Exceptions can be specified by adding ignore rules to `ignore.txt`. (This is even necessary for apps like Facebook Messenger and App Store, which refuse to connect through a MITM proxy.) **Traffic to and from hosts matched by such rules _cannot_ be read or modified by mitmproxy or Userscript Proxy.** ## Data usage From e94010e6c535ec3a309f237deb12b223cdc6e7e4 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Fri, 29 Dec 2017 23:12:17 +0100 Subject: [PATCH 031/103] Fix encoding issues This commit fixes two issues: 1. The charset was being wrongly extracted from the Content-Type header. 2. soup.prettify() apparently broke Facebook. --- injector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/injector.py b/injector.py index 0b95680..af5c360 100644 --- a/injector.py +++ b/injector.py @@ -63,7 +63,7 @@ def indexOfDTD(soup: BeautifulSoup) -> Optional[int]: def inferEncoding(response: http.HTTPResponse) -> Optional[str]: httpHeaderValue = response.headers.get(CONTENT_TYPE, "").lower() match = REGEX_CHARSET.search(httpHeaderValue) - return match.group(0) if match else None + return match.group(1) if match else None class UserscriptInjector: def __init__(self): @@ -171,7 +171,7 @@ def response(self, flow: http.HTTPFlow): # There is a DTD and it is invalid, so replace it. soup.contents[index_DTD] = Doctype(re.sub(REGEX_DOCTYPE, "", soup.contents[index_DTD])) # Serialize and encode: - response.content = soup.prettify().encode( + response.content = str(soup).encode( fromOptional(soup.original_encoding, CHARSET_DEFAULT), "replace" ) From 99a4ebf4ea8977f59f2abcd37a0012c931a7c85d Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Fri, 29 Dec 2017 23:46:03 +0100 Subject: [PATCH 032/103] Add lib/ to typecheck script --- typecheck | 1 + 1 file changed, 1 insertion(+) diff --git a/typecheck b/typecheck index bbde462..884600d 100644 --- a/typecheck +++ b/typecheck @@ -4,6 +4,7 @@ export MYPYPATH=/usr/local/lib/python3.6/dist-packages/ if [ "$1" == "" ]; then mypy *.py --ignore-missing-imports --follow-imports skip + mypy lib/*.py --ignore-missing-imports --follow-imports skip else mypy $1 --ignore-missing-imports --follow-imports skip fi From 45fcec0006f405fceea53a0b7ba3e33cfa1633f4 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Fri, 29 Dec 2017 23:46:54 +0100 Subject: [PATCH 033/103] Bump version number to 0.4.1 --- injector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/injector.py b/injector.py index af5c360..e03cfbe 100644 --- a/injector.py +++ b/injector.py @@ -13,7 +13,7 @@ def stringifyVersion(version: str) -> str: return VERSION_PREFIX + version -VERSION: str = "0.4.0" +VERSION: str = "0.4.1" VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" WELCOME_MESSAGE: str = APP_NAME + " " + stringifyVersion(VERSION) From f09abc73299b5b45e17849f4141653c1d601f83b Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sat, 30 Dec 2017 13:56:11 +0100 Subject: [PATCH 034/103] Add ignore rules for Akamai and Google CDNs --- ignore.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ignore.txt b/ignore.txt index 86555f5..6649080 100644 --- a/ignore.txt +++ b/ignore.txt @@ -31,6 +31,8 @@ # Traffic irrelevant to Userscript Proxy: /cdn\./ + akamai*.net # Akamai CDN + googleusercontent.com # Google CDN i.ytimg.com # YouTube thumbnails *.googlevideo.com # YouTube video content api.twitch.tv # Twitch app metadata From a59cdc93c07a9e15d0e2b4439def97434b95f7a0 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sat, 31 Mar 2018 10:17:34 +0200 Subject: [PATCH 035/103] Add ignore rules: Dropbox, Apple, Crashlytics --- ignore.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ignore.txt b/ignore.txt index 6649080..695671a 100644 --- a/ignore.txt +++ b/ignore.txt @@ -28,6 +28,10 @@ api.facebook.com # Messenger edge-mqtt.facebook.com # Messenger slack.com # Slack + api*.dropbox.com # Dropbox app + bolt.dropbox.com # Dropbox app + ls.apple.com # Apple services + *.crashlytics.com # Crashlytics error reporting # Traffic irrelevant to Userscript Proxy: /cdn\./ From 44fa0bb44a499d4652608980c1d976f5fc5c62c8 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 3 Jun 2018 22:45:50 +0200 Subject: [PATCH 036/103] Fix "Un-loading script" bug (mitmproxy 3+) The addons API used in mitmproxy 3+ required that this update be done. --- injector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/injector.py b/injector.py index e03cfbe..4484e05 100644 --- a/injector.py +++ b/injector.py @@ -107,6 +107,7 @@ def __init__(self): logError("Userscript error:") logError(str(err)) continue + os.chdir("..") # so mitmproxy does not unload the script logInfo("") logInfo(str(len(loadedUserscripts)) + " userscript(s) loaded:") @@ -177,5 +178,4 @@ def response(self, flow: http.HTTPFlow): ) -def start(): - return UserscriptInjector() +addons = [ UserscriptInjector() ] From 7579da6caf1c9ac2d77386e01483aa504b56ce92 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 3 Jun 2018 23:41:25 +0200 Subject: [PATCH 037/103] Fix type errors --- lib/metadata.py | 40 ++++++++++++++++++++++++++-------------- lib/patterns.py | 5 +++-- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/lib/metadata.py b/lib/metadata.py index 3188f70..ea7af53 100644 --- a/lib/metadata.py +++ b/lib/metadata.py @@ -128,12 +128,18 @@ def parseLine(line: str) -> Optional[MetadataItem]: else: tagName: str = match.group(REGEXGROUP_TAGNAME) tagValue: Optional[str] = match.group(REGEXGROUP_TAGVALUE) - return (tagName, tagValue if isSomething(tagValue) else True) # Boolean tags have no explicit value; if they are present, they are true. + # Boolean tags have no explicit value; if they are present, they are true: + if tagValue is None: + return (tagName, True) + else: + return (tagName, tagValue) - return list(filter( - isSomething, - map(parseLine, metadataContent.splitlines()) - )) + # filter did not play well with mypy: + parsedItems: Metadata = [] + for item in map(parseLine, metadataContent.splitlines()): + if item is not None: + parsedItems.append(item) + return parsedItems def tagByName(tags: List[Tag], tagName: str) -> Optional[Tag]: @@ -142,7 +148,7 @@ def tagByName(tags: List[Tag], tagName: str) -> Optional[Tag]: def validatePair(tags: List[Tag], pair: MetadataItem) -> MetadataItem: (tagName, tagValue) = pair - tag: Tag = tagByName(tags, tagName) + tag: Optional[Tag] = tagByName(tags, tagName) if tag is None: # Unrecognized key. return (tagName, tagValue) @@ -153,7 +159,7 @@ def validatePair(tags: List[Tag], pair: MetadataItem) -> MetadataItem: raise MetadataError(STRING_ERROR_MISSING_VALUE.substitute(tagName=tagName)) if type(tag) is Tag_boolean: tagValue = tagValue is not False # This handles cases like `@noframes blabla`; a boolean directive is true no matter what comes after it. - if isSomething(tagPredicate): + if tagPredicate is not None: if not tagPredicate(tagValue): raise MetadataError(STRING_ERROR_PREDICATE_FAILED.substitute(tagName=tagName, tagValue=str(tagValue))) return (tagName, tagValue) @@ -162,22 +168,28 @@ def validatePair(tags: List[Tag], pair: MetadataItem) -> MetadataItem: def validate(tags: List[Tag], metadata: Metadata) -> Metadata: # raises MetadataError def handleDuplicate(acc: Iterable[MetadataItem], pair: MetadataItem) -> Iterable[MetadataItem]: name: str = first(pair) - tag: Tag = tagByName(tags, name) + tag: Optional[Tag] = tagByName(tags, name) seenTagNames: Iterator[str] = map(first, acc) # Throw away pair if it has the same tag name as some already seen, known, unique directive: - return acc if isSomething(tag) and tag.unique and name in seenTagNames else list(acc) + [pair] + return acc if tag is not None and tag.unique and name in seenTagNames else list(acc) + [pair] def withoutDuplicates(metadata: Metadata) -> Metadata: return list(reduce(handleDuplicate, metadata, [])) + # Awkwardly written to satisfy mypy: def withDefaults(metadata: Metadata) -> Metadata: tagNamesThatWeHave: List[str] = list(map(first, metadata)) - def hasDefaultAndNotAlreadyParsed(tag: Tag) -> bool: - return isSomething(tag.default) and tag.name not in tagNamesThatWeHave - neededDefaults: Metadata = list(map( + unseenItems = map( lambda tag: (tag.name, tag.default), - filter(hasDefaultAndNotAlreadyParsed, tags) - )) + filter( + lambda tag: tag.name not in tagNamesThatWeHave, + tags + ) + ) + neededDefaults: Metadata = [] + for (tagName, default) in unseenItems: + if default is not None: + neededDefaults.append((tagName, default)) return metadata + neededDefaults def assertRequiredPresent(metadata: Metadata) -> Metadata: diff --git a/lib/patterns.py b/lib/patterns.py index 1be121c..dea8585 100644 --- a/lib/patterns.py +++ b/lib/patterns.py @@ -1,4 +1,4 @@ -from typing import Optional, Pattern +from typing import Optional, Pattern, Match import re from lib.utilities import first, isSomething @@ -63,7 +63,8 @@ def regexify(segment: str) -> str: # Returns None if the pattern is invalid: def extractGroup(group: str, matchPattern: str) -> Optional[str]: try: - return REGEX_MATCH_PATTERN.search(normalizeMatchPattern(matchPattern)).group(group) + match: Optional[Match] = REGEX_MATCH_PATTERN.search(normalizeMatchPattern(matchPattern)) + return None if match is None else match.group(group) except: return None From f42cacd5609c64c4360904bfc6a584e8e393a9db Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 3 Jun 2018 23:50:58 +0200 Subject: [PATCH 038/103] Bump version number to 0.5.0 --- injector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/injector.py b/injector.py index 4484e05..cf0a355 100644 --- a/injector.py +++ b/injector.py @@ -13,7 +13,7 @@ def stringifyVersion(version: str) -> str: return VERSION_PREFIX + version -VERSION: str = "0.4.1" +VERSION: str = "0.5.0" VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" WELCOME_MESSAGE: str = APP_NAME + " " + stringifyVersion(VERSION) From 5b639544b758b1c062250a1fb65c5fc5f9ff72fd Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 13 Aug 2018 19:36:54 +0200 Subject: [PATCH 039/103] Remove bad exception handling Not all errors have to do with files being unreadable. --- userscript-proxy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/userscript-proxy.py b/userscript-proxy.py index da5f872..3ac7ab0 100644 --- a/userscript-proxy.py +++ b/userscript-proxy.py @@ -21,5 +21,3 @@ print("Interrupted by user.") except PermissionError: print("Could not read file `"+FILENAME_IGNORE+"`: Permission denied.") -except Exception as e: - print("Could not read file `"+FILENAME_IGNORE+"`: " + str(e)) From 6c48d5063bd43d664a09e6ef7019e30233d0c206 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Tue, 14 Aug 2018 10:47:19 +0200 Subject: [PATCH 040/103] Implement injection by link Instead of injecting tens or hundreds of kilobytes of JavaScript into every single response, we can now just insert a `), never linked (``). Useful to test new userscript features without having to re-upload the userscript and clear browser cache. + +### `--port PORT` + +Make mitmproxy listen to TCP port PORT. Defaults to 8080. + +### `--transparent` + +Run mitmproxy in [transparent mode](transparent-mode). Useful if you cannot set a proxy in the client. In such cases, you may have to route traffic from the client to the proxy at the network layer instead, making transparent mode necessary. + +### `--verbose` + +Inject a comment in each page specifying which userscripts (if any) were injected. + + [mitmproxy]: https://mitmproxy.org [minification]: https://en.wikipedia.org/wiki/Minification_(programming) [metadata]: https://wiki.greasespot.net/Metadata_Block +[transparent-mode]: https://docs.mitmproxy.org/stable/concepts-modes/#transparent-proxy From 010f9962803a2676bd1c18a723c6502b0bc851f1 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 16 Sep 2018 15:46:01 +0200 Subject: [PATCH 059/103] Make typecheck script more portable --- typecheck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typecheck b/typecheck index 46f0083..3651802 100755 --- a/typecheck +++ b/typecheck @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash export MYPYPATH=/usr/local/lib/python3.6/dist-packages/ From 36c3a5375e9aa2ab8b5888b3471d8021f7f9fd46 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Sun, 16 Sep 2018 15:46:30 +0200 Subject: [PATCH 060/103] v0.8.0 --- modules/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/constants.py b/modules/constants.py index 5c6a82c..a029e9f 100644 --- a/modules/constants.py +++ b/modules/constants.py @@ -1,5 +1,5 @@ VERSION_PREFIX: str = "v" APP_NAME: str = "Userscript Proxy" -VERSION: str = "0.7.1" +VERSION: str = "0.8.0" ATTRIBUTE_UP_VERSION: str = "data-userscript-proxy-version" DEFAULT_PORT: int = 8080 From 5fd30f8908237e9fd9ab2121941d64aa750f5647 Mon Sep 17 00:00:00 2001 From: Simon Alling Date: Mon, 17 Sep 2018 12:14:18 +0200 Subject: [PATCH 061/103] =?UTF-8?q?Readme:=20Clarify=20linked=E2=80=93inli?= =?UTF-8?q?ne=20mutual=20exclusivity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 690bc29..d6ef7b5 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Exceptions can be specified by adding ignore rules to `ignore.txt`. (This is eve ## Data usage UP has no data usage impact when no userscript is injected, i.e. for URLs without any matching userscript. -When a script _is_ injected, exactly one of the following things happens: +When a script _is_ injected, **exactly one** of the following things happens: * The entire userscript is injected as inline JavaScript (potentially dozens or even hundreds of kilobytes). * A `