From 0f1ade94a227ac31262a883e7bbcf336e398b96c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kevin=20R=C3=B6bert?= <dev@xn--rb-fka.it>
Date: Tue, 30 Jul 2019 20:29:41 +0200
Subject: [PATCH] Version 1.7.0

+ Added support for raw rules to cleaning also parts from URL-path
---
 CHANGELOG.md       |  9 ++++++
 clearurls.js       | 79 +++++++++++++++++++++++++++++++++++++++++++++-
 data/data.min.json | 54 +++++++++++++++++++++++++++++++
 html/popup.html    |  2 +-
 manifest.json      |  2 +-
 5 files changed, 143 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c8505fe..26d5b3a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.7.0] - 2019-07-30
+
+### Compatibility note
+- Require Firefox >= 55
+- Require Chrome >= 22
+
+### Added
+- Added support for raw rules to cleaning also parts from URL-path
+
 ## [1.6.9] - 2019-07-29
 
 ### Compatibility note
diff --git a/clearurls.js b/clearurls.js
index 771ad31..bf11cb6 100644
--- a/clearurls.js
+++ b/clearurls.js
@@ -46,6 +46,40 @@ function removeFieldsFormURL(provider, pureUrl)
     var rules = provider.getRules();
     var changes = false;
     var cancel = false;
+    var rawRules = provider.getRawRules();
+
+    /*
+    * Apply raw rules to the URL.
+    */
+    rawRules.forEach(function(rawRule) {
+        var beforReplace = url;
+        url = url.replace(new RegExp(rawRule, "gi"), "");
+
+        if(beforReplace !== url) {
+            //Log the action
+            if(storage.loggingStatus)
+            {
+                pushToLog(beforReplace, url, rawRule);
+            }
+
+            if(badges[tabid] == null) badges[tabid] = 0;
+
+            increaseURLCounter();
+
+            if(!checkOSAndroid())
+            {
+                if(storage.badgedStatus) {
+                    browser.browserAction.setBadgeText({text: (++badges[tabid]).toString(), tabId: tabid});
+                }
+                else
+                {
+                    browser.browserAction.setBadgeText({text: "", tabId: tabid});
+                }
+            }
+
+            changes = true;
+        }
+    });
 
     if(existsFragments(url)) {
         domain = url.replace(new RegExp("#.*", "i"), "");
@@ -71,7 +105,6 @@ function removeFieldsFormURL(provider, pureUrl)
         };
     }
 
-
     if(existsFields(url)) {
         fields = "?"+extractFileds(url).rmEmpty().join("&");
     }
@@ -206,6 +239,12 @@ function start()
                     providers[p].addRule(data.providers[prvKeys[p]].rules[r]);
                 }
 
+                //Add raw rules to provider
+                for(var raw = 0; raw < data.providers[prvKeys[p]].rawRules.length; raw++)
+                {
+                    providers[p].addRawRule(data.providers[prvKeys[p]].rawRules[raw]);
+                }
+
                 //Add exceptions to provider
                 for(var e = 0; e < data.providers[prvKeys[p]].exceptions.length; e++)
                 {
@@ -321,6 +360,8 @@ function start()
             var enabled_redirections = {};
             var disabled_redirections = {};
             var active = _isActive;
+            var enabled_rawRules = {};
+            var disabled_rawRules = {};
 
             if(_completeProvider){
                 enabled_rules[".*"] = true;
@@ -398,6 +439,42 @@ function start()
                 return Object.keys(enabled_rules);
             };
 
+            /**
+            * Add a raw rule to the raw rule array
+            * and replace old raw rule with new raw rule.
+            *
+            * @param {String} rule        RegExp as string
+            * @param {boolean} isActive   Is this rule active?
+            */
+            this.addRawRule = function(rule, isActive = true) {
+                if(isActive)
+                {
+                    enabled_rawRules[rule] = true;
+
+                    if(disabled_rawRules[rule] !== undefined)
+                    {
+                        delete disabled_rawRules[rule];
+                    }
+                }
+                else {
+                    disabled_rawRules[rule] = true;
+
+                    if(enabled_rawRules[rule] !== undefined)
+                    {
+                        delete enabled_rawRules[rule];
+                    }
+                }
+            };
+
+            /**
+            * Return all active raw rules as an array.
+            *
+            * @return Array RegExp strings
+            */
+            this.getRawRules = function() {
+                return Object.keys(enabled_rawRules);
+            };
+
             /**
             * Add a exception to the exceptions array
             * and replace old with new exception.
diff --git a/data/data.min.json b/data/data.min.json
index 02f9f43..4fcb30e 100644
--- a/data/data.min.json
+++ b/data/data.min.json
@@ -26,12 +26,16 @@
             "exceptions": [
                 ".*(amazon\\.).*(\\/gp).*\\/redirector.html\\/.*"
             ],
+            "rawRules": [
+                "\\/ref=[^\\/|\\?]*"
+            ],
             "redirections": []
         },
         "fls-na.amazon": {
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(fls-na\\.amazon)(\\.[a-zA-Z]{2,}).*",
             "completeProvider": true,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -69,6 +73,7 @@
                 "cad",
                 "uact"
             ],
+            "rawRules": [],
             "exceptions": [
                 ".*(mail\\.google\\.).*(\\/mail\\/u\\/0).*",
                 ".*(google\\.).*(\\/upload)?(\\/drive)\\/.*",
@@ -93,6 +98,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(googlesyndication)(\\.[a-zA-Z]{2,}).*",
             "completeProvider": true,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -100,6 +106,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(doubleclick)(\\.[a-zA-Z]{2,}).*",
             "completeProvider": true,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*doubleclick\\..*\\/.*tag_for_child_directed_treatment=;%3F(.*)"
@@ -109,6 +116,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(googleadservices)(\\.[a-zA-Z]{2,}).*",
             "completeProvider": true,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*googleadservices\\..*\\/.*adurl=([^&]*)"
@@ -142,6 +150,7 @@
                 "(%3F)?hmb_source",
                 "(%3F)?[\\?|&]ref[\\_]?"
             ],
+            "rawRules": [],
             "exceptions": [
                 ".*([\\.]?matrix\\.org)(\\/_matrix)\\/.*",
                 ".*([\\.]?prismic\\.io).*",
@@ -157,6 +166,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(adtech)(\\.[a-zA-Z]{2,}).*",
             "completeProvider": true,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -164,6 +174,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(contentpass\\.net).*",
             "completeProvider": true,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -171,6 +182,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(bf-ad)(\\.[a-zA-Z]{2,}).*",
             "completeProvider": true,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -178,6 +190,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(amazon-adsystem)(\\.[a-zA-Z]{2,}).*",
             "completeProvider": true,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -185,6 +198,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(adsensecustomsearchads)(\\.[a-zA-Z]{2,}).*",
             "completeProvider": true,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -196,6 +210,7 @@
                 "gclid",
                 "kw"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*youtube\\..*\\/redirect?.*q=([^&]*)"
@@ -222,6 +237,7 @@
                 "acontext",
                 "action_history"
             ],
+            "rawRules": [],
             "exceptions": [
                 ".*(facebook\\.)\\w{2,}.*(\\/plugins\\/).*",
                 ".*(facebook\\.)\\w{2,}.*(\\/dialog\\/share).*"
@@ -237,6 +253,7 @@
                 "(ref_)?src",
                 "s"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -244,6 +261,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(reddit)(\\.[a-zA-Z]{2,}).*",
             "completeProvider": false,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*out\\.reddit\\.\\w{2,}\\/.*url=([^&]*)"
@@ -258,6 +276,7 @@
                 "tctx",
                 "jb[a-zA-Z]*"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -269,6 +288,7 @@
                 "sr",
                 "sr_share"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -284,6 +304,7 @@
                 "qs",
                 "qp"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -294,6 +315,7 @@
                 "nb",
                 "u"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -304,6 +326,7 @@
                 "tt_medium",
                 "tt_content"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -314,6 +337,7 @@
                 "pk_campaign",
                 "pk_kwd"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -325,6 +349,7 @@
                 "alid",
                 "[a-zA-Z]*tk"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -341,6 +366,7 @@
                 "exp",
                 "plim"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -352,6 +378,7 @@
                 "_trksid",
                 "_from"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -361,6 +388,7 @@
             "rules": [
                 "ftag"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -371,6 +399,7 @@
                 "ref_",
                 "pf_rd_[a-zA-Z]*"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -378,6 +407,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(govdelivery\\.com).*",
             "completeProvider": false,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*links\\.govdelivery\\.com.*\\/track\\?.*(http:\\/\\/.*)",
@@ -391,6 +421,7 @@
                 "u1",
                 "ath[a-zA-Z]*"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -400,6 +431,7 @@
             "rules": [
                 "pl"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -409,6 +441,7 @@
             "rules": [
                 "xid"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -419,6 +452,7 @@
                 "spm",
                 "tpa"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -428,6 +462,7 @@
             "rules": [
                 "smid"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -437,6 +472,7 @@
             "rules": [
                 "wbdcd"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -446,6 +482,7 @@
             "rules": [
                 "snr"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -453,6 +490,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(steamcommunity\\.com).*",
             "completeProvider": false,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*steamcommunity\\.com.*\\/linkfilter\\/\\?url=(.*)"
@@ -462,6 +500,7 @@
             "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-]*\\.)?(disq\\.us).*",
             "completeProvider": false,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*disq\\.us.*\\/.*url\\?.*url=((https%3A%2F%2F|http%3A%2F%2F).*)%3A"
@@ -471,6 +510,7 @@
             "urlPattern": "https?:\\/\\/outgoing\\.prod\\.mozaws\\.net/.*",
             "completeProvider": false,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 "https?:\\/\\/[^/]+/v1/[0-9a-f]{64}/(.*)"
@@ -482,6 +522,7 @@
             "rules": [
                 "src"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -492,6 +533,7 @@
                 "src",
                 "platform"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -501,6 +543,7 @@
             "rules": [
                 "ref"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -510,6 +553,7 @@
             "rules": [
                 "email"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -520,6 +564,7 @@
                 "email_token",
                 "email_source"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -527,6 +572,7 @@
             "urlPattern": "https?:\\/\\/([a-zA-Z0-9-]*\\.)?(deviantart\\.com).*",
             "completeProvider": false,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*deviantart\\.com.*outgoing\\?(.*)"
@@ -536,6 +582,7 @@
             "urlPattern": "https?:\\/\\/([a-zA-Z0-9-]*\\.)?(site2\\.com).*",
             "completeProvider": false,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*site2\\.com.*\\?.*=(.*)"
@@ -545,6 +592,7 @@
             "urlPattern": "https?:\\/\\/([a-zA-Z0-9-]*\\.)?(site\\.com).*",
             "completeProvider": false,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*site\\.com.*\\?to=([^&]*)"
@@ -554,6 +602,7 @@
             "urlPattern": "https?:\\/\\/([a-zA-Z0-9-]*\\.)?(site3\\.com).*",
             "completeProvider": false,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*site3\\.com.*\\?r=([^&]*)"
@@ -568,6 +617,7 @@
                 "algo_expid",
                 "algo_pvid"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -577,6 +627,7 @@
             "rules": [
                 "sid"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -586,6 +637,7 @@
             "rules": [
                 "ref"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         },
@@ -593,6 +645,7 @@
             "urlPattern": "https?:\\/\\/([a-zA-Z0-9-]*\\.)?(linksynergy\\.com).*",
             "completeProvider": false,
             "rules": [],
+            "rawRules": [],
             "exceptions": [],
             "redirections": [
                 ".*linksynergy\\.com.*\\/.*murl=([^&]*)"
@@ -604,6 +657,7 @@
             "rules": [
                 "ref"
             ],
+            "rawRules": [],
             "exceptions": [],
             "redirections": []
         }
diff --git a/html/popup.html b/html/popup.html
index 5129f09..6d1a702 100644
--- a/html/popup.html
+++ b/html/popup.html
@@ -137,7 +137,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
             <div class="col-sm-1">
                 <h5><b id="rules_status_head"></b></h5>
                 <div class="text-center">
-                    <a href="https://gitlab.com/KevinRoebert/ClearUrls/commits/master/data/data.json"
+                    <a href="https://gitlab.com/KevinRoebert/ClearUrls/commits/master/data/data.min.json"
                     id="hashStatus" class="btn btn-primary btn-xs" target="_blank"></a>
                 </div>
                 <div class="clearfix"></div>
diff --git a/manifest.json b/manifest.json
index bccd7f0..f313d18 100644
--- a/manifest.json
+++ b/manifest.json
@@ -1,7 +1,7 @@
 {
     "manifest_version": 2,
     "name": "ClearURLs",
-    "version": "1.6.9",
+    "version": "1.7.0",
     "author": "Kevin R.",
     "description": "Remove tracking elements from URLs.",
     "homepage_url": "https://gitlab.com/KevinRoebert/ClearUrls",