diff --git a/clearurls.js b/clearurls.js
index bf4d388..6f1f6ea 100644
--- a/clearurls.js
+++ b/clearurls.js
@@ -1,685 +1,685 @@
-/*
-* ClearURLs
-* Copyright (c) 2017-2020 Kevin Röbert
-*
-* This program is free software: you can redistribute it and/or modify
-* it under the terms of the GNU Lesser General Public License as published by
-* the Free Software Foundation, either version 3 of the License, or
-* (at your option) any later version.
-*
-* This program is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-* GNU Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Lesser General Public License
-* along with this program. If not, see .
-*/
-
-/*jshint esversion: 6 */
-/*
-* This script is responsible for the core functionalities.
-*/
-var providers = [];
-var prvKeys = [];
-var siteBlockedAlert = 'javascript:void(0)';
-var dataHash;
-var localDataHash;
-var os;
-
-/**
- * Helper function which remove the tracking fields
- * for each provider given as parameter.
- *
- * @param {Provider} provider Provider-Object
- * @param pureUrl URL as String
- * @param {boolean} quiet if the action should be displayed in log and statistics
- * @param {requestDetails} request the request details
- * @return {Array} Array with changes and url fields
- */
-function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) {
- let url = pureUrl;
- let domain = "";
- let fragments = "";
- let fields = "";
- let rules = provider.getRules();
- let changes = false;
- let cancel = false;
- let rawRules = provider.getRawRules();
-
- if (storage.localHostsSkipping && checkLocalURL(pureUrl)) {
- return {
- "changes": false,
- "url": url,
- "cancel": false
- };
- }
-
- /*
- * Apply raw rules to the URL.
- */
- rawRules.forEach(function (rawRule) {
- let beforeReplace = url;
- url = url.replace(new RegExp(rawRule, "gi"), "");
-
- if (beforeReplace !== url) {
- //Log the action
- if (storage.loggingStatus && !quiet) {
- pushToLog(beforeReplace, url, rawRule);
- }
-
- increaseBadged(quiet, request);
- changes = true;
- }
- });
-
- if (existsFragments(url)) {
- domain = url.replace(new RegExp("#.*", "i"), "");
- }
- if (existsFields(url)) {
- domain = url.replace(new RegExp("\\?.*", "i"), "");
- }
-
- /*
- * Expand the url by provider redirections. So no tracking on
- * url redirections form sites to sites.
- */
- let re = provider.getRedirection(url);
- if (re !== null) {
- url = decodeURL(re);
-
- //Log the action
- if (!quiet) {
- pushToLog(pureUrl, url, translate('log_redirect'));
- increaseGlobalURLCounter(1);
- increaseBadged(false, request)
- };
-
- return {
- "redirect": true,
- "url": url
- };
- }
-
- if (existsFields(url)) {
- fields = "?" + extractFileds(url).rmEmpty().join("&");
- }
-
- if (existsFragments(url)) {
- fragments = "#" + extractFragments(url).rmEmpty().join("&");
- }
-
- /**
- * Only test for matches, if there are fields or fragments that can be cleaned.
- */
- if (fields !== "" || fragments !== "") {
- rules.forEach(function (rule) {
- let beforeReplace = fields;
- let beforeReplaceFragments = fragments;
- fields = fields.replace(new RegExp(rule, "gi"), "");
- fragments = fragments.replace(new RegExp(rule, "gi"), "");
-
- if (beforeReplace !== fields || beforeReplaceFragments !== fragments) {
- //Log the action
- if (storage.loggingStatus) {
- let tempURL = domain;
- let tempBeforeURL = domain;
-
- if (fields !== "") tempURL += "?" + fields.replace("?&", "?").replace("?", "");
- if (fragments !== "") tempURL += "#" + fragments.replace("#&", "#").replace("#", "");
- if (beforeReplace !== "") tempBeforeURL += "?" + beforeReplace.replace("?&", "?").replace("?", "");
- if (beforeReplaceFragments !== "") tempBeforeURL += "#" + beforeReplaceFragments.replace("#&", "#").replace("#", "");
-
- if (!quiet) pushToLog(tempBeforeURL, tempURL, rule);
- }
-
- increaseBadged(quiet, request);
- changes = true;
- }
- });
-
- let finalURL = domain;
-
- if (fields !== "") finalURL += "?" + fields.replace("?", "");
- if (fragments !== "") finalURL += "#" + fragments.replace("#", "");
-
- url = finalURL.replace(new RegExp("\\?&"), "?").replace(new RegExp("#&"), "#");
- }
-
- if (provider.isCaneling() && storage.domainBlocking) {
- if (!quiet) pushToLog(pureUrl, pureUrl, translate('log_domain_blocked'));
- increaseGlobalURLCounter(1);
- increaseBadged(quiet, request);
- cancel = true;
- }
-
- return {
- "changes": changes,
- "url": url,
- "cancel": cancel
- };
-}
-
-function start() {
- /**
- * Initialize the JSON provider object keys.
- *
- * @param {object} obj
- */
- function getKeys(obj) {
- for (const key in obj) {
- prvKeys.push(key);
- }
- }
-
- /**
- * Initialize the providers form the JSON object.
- *
- */
- function createProviders() {
- let data = storage.ClearURLsData;
-
- for (let p = 0; p < prvKeys.length; p++) {
- //Create new provider
- providers.push(new Provider(prvKeys[p], data.providers[prvKeys[p]].getOrDefault('completeProvider', false),
- data.providers[prvKeys[p]].getOrDefault('forceRedirection', false)));
-
- //Add URL Pattern
- providers[p].setURLPattern(data.providers[prvKeys[p]].getOrDefault('urlPattern', ''));
-
- let rules = data.providers[prvKeys[p]].getOrDefault('rules', []);
- //Add rules to provider
- for (let r = 0; r < rules.length; r++) {
- providers[p].addRule(rules[r]);
- }
-
- let rawRules = data.providers[prvKeys[p]].getOrDefault('rawRules', []);
- //Add raw rules to provider
- for (let raw = 0; raw < rawRules.length; raw++) {
- providers[p].addRawRule(rawRules[raw]);
- }
-
- let referralMarketingRules = data.providers[prvKeys[p]].getOrDefault('referralMarketing', []);
- //Add referral marketing rules to provider
- for (let referralMarketing = 0; referralMarketing < referralMarketingRules.length; referralMarketing++) {
- providers[p].addReferralMarketing(referralMarketingRules[referralMarketing]);
- }
-
- let exceptions = data.providers[prvKeys[p]].getOrDefault('exceptions', []);
- //Add exceptions to provider
- for (let e = 0; e < exceptions.length; e++) {
- providers[p].addException(exceptions[e]);
- }
-
- let redirections = data.providers[prvKeys[p]].getOrDefault('redirections', []);
- //Add redirections to provider
- for (let re = 0; re < redirections.length; re++) {
- providers[p].addRedirection(redirections[re]);
- }
- }
- }
-
- /**
- * Convert the external data to Objects and
- * call the create provider function.
- *
- * @param {String} retrievedText - pure data form github
- */
- function toObject(retrievedText) {
- getKeys(storage.ClearURLsData.providers);
- createProviders();
- }
-
- /**
- * Deactivates ClearURLs, if no rules can be downloaded and also no old rules in storage
- */
- function deactivateOnFailure() {
- if(storage.ClearURLsData.length === 0) {
- storage.globalStatus = false;
- storage.dataHash = "";
- changeIcon();
- storeHashStatus(5);
- saveOnExit();
- }
- }
-
- /**
- * Get the hash for the rule file on GitLab.
- * Check the hash with the hash form the local file.
- * If the hash has changed, then download the new rule file.
- * Else do nothing.
- */
- function getHash() {
- //Get the target hash from GitLab
- fetch(storage.hashURL)
- .then(function (response) {
- const responseTextHash = response.clone().text().then(function (responseTextHash) {
- if (response.ok && $.trim(responseTextHash)) {
- dataHash = responseTextHash;
-
- if ($.trim(dataHash) !== $.trim(localDataHash)) {
- fetchFromURL();
- } else {
- toObject(storage.ClearURLsData);
- storeHashStatus(1);
- saveOnDisk(['hashStatus']);
- }
- } else {
- dataHash = false;
- deactivateOnFailure();
- }
- });
- });
- }
-
- /*
- * ##################################################################
- * # Fetch Rules & Exception from URL #
- * ##################################################################
- */
- function fetchFromURL() {
- fetch(storage.ruleURL)
- .then(checkResponse);
-
- function checkResponse(response) {
- const responseText = response.clone().text().then(function (responseText) {
- if (response.ok && $.trim(responseText)) {
- const downloadedFileHash = $.sha256(responseText);
-
- if ($.trim(downloadedFileHash) === $.trim(dataHash)) {
- storage.ClearURLsData = responseText;
- storage.dataHash = downloadedFileHash;
- storeHashStatus(2);
- } else {
- storeHashStatus(3);
- }
- storage.ClearURLsData = JSON.parse(storage.ClearURLsData);
- toObject(storage.ClearURLsData);
- saveOnDisk(['ClearURLsData', 'dataHash', 'hashStatus']);
- } else {
- deactivateOnFailure();
- }
- });
- }
- }
-
- // ##################################################################
-
- /*
- * ##################################################################
- * # Supertyp Provider #
- * ##################################################################
- */
- /**
- * Declare constructor
- *
- * @param {String} _name Provider name
- * @param {boolean} _completeProvider Set URL Pattern as rule
- * @param {boolean} _forceRedirection Whether redirects should be enforced via a "tabs.update"
- * @param {boolean} _isActive Is the provider active?
- */
- function Provider(_name, _completeProvider = false, _forceRedirection = false, _isActive = true) {
- let name = _name;
- let urlPattern;
- let enabled_rules = {};
- let disabled_rules = {};
- let enabled_exceptions = {};
- let disabled_exceptions = {};
- let canceling = _completeProvider;
- let enabled_redirections = {};
- let disabled_redirections = {};
- let active = _isActive;
- let enabled_rawRules = {};
- let disabled_rawRules = {};
- let enabled_referralMarketing = {};
- let disabled_referralMarketing = {};
-
- if (_completeProvider) {
- enabled_rules[".*"] = true;
- }
-
- /**
- * Returns whether redirects should be enforced via a "tabs.update"
- * @return {boolean} whether redirects should be enforced
- */
- this.shouldForceRedirect = function () {
- return _forceRedirection;
- };
-
- /**
- * Returns the provider name.
- * @return {String}
- */
- this.getName = function () {
- return name;
- };
-
- /**
- * Add URL pattern.
- *
- * @require urlPatterns as RegExp
- */
- this.setURLPattern = function (urlPatterns) {
- urlPattern = new RegExp(urlPatterns, "i");
- };
-
- /**
- * Return if the Provider Request is canceled
- * @return {Boolean} isCanceled
- */
- this.isCaneling = function () {
- return canceling;
- };
-
- /**
- * Check the url is matching the ProviderURL.
- *
- * @return {boolean} ProviderURL as RegExp
- */
- this.matchURL = function (url) {
- return urlPattern.test(url) && !(this.matchException(url));
- };
-
- /**
- * Apply a rule to a given tuple of rule array.
- * @param enabledRuleArray array for enabled rules
- * @param disabledRulesArray array for disabled rules
- * @param {String} rule RegExp as string
- * @param {boolean} isActive Is this rule active?
- */
- this.applyRule = (enabledRuleArray, disabledRulesArray, rule, isActive = true) => {
- if (isActive) {
- enabledRuleArray[rule] = true;
-
- if (disabledRulesArray[rule] !== undefined) {
- delete disabledRulesArray[rule];
- }
- } else {
- disabledRulesArray[rule] = true;
-
- if (enabledRuleArray[rule] !== undefined) {
- delete enabledRuleArray[rule];
- }
- }
- };
-
- /**
- * Add a rule to the rule array
- * and replace old rule with new rule.
- *
- * @param {String} rule RegExp as string
- * @param {boolean} isActive Is this rule active?
- */
- this.addRule = function (rule, isActive = true) {
- rule = "([\\/\\?#]|(&|&))+(" + rule + "=[^&]*)";
-
- this.applyRule(enabled_rules, disabled_rules, rule, isActive);
- };
-
- /**
- * Return all active rules as an array.
- *
- * @return Array RegExp strings
- */
- this.getRules = function () {
- if (!storage.referralMarketing) {
- return Object.keys(Object.assign(enabled_rules, enabled_referralMarketing));
- }
-
- return Object.keys(enabled_rules);
- };
-
- /**
- * Add a raw rule to the raw rule array
- * and replace old raw rule with new raw rule.
- *
- * @param {String} rule RegExp as string
- * @param {boolean} isActive Is this rule active?
- */
- this.addRawRule = function (rule, isActive = true) {
- this.applyRule(enabled_rawRules, disabled_rawRules, rule, isActive);
- };
-
- /**
- * Return all active raw rules as an array.
- *
- * @return Array RegExp strings
- */
- this.getRawRules = function () {
- return Object.keys(enabled_rawRules);
- };
-
- /**
- * Add a referral marketing rule to the referral marketing array
- * and replace old referral marketing rule with new referral marketing rule.
- *
- * @param {String} rule RegExp as string
- * @param {boolean} isActive Is this rule active?
- */
- this.addReferralMarketing = function (rule, isActive = true) {
- rule = "([\\/\\?#]|(&|&))+(" + rule + "=[^\\/\\?&]*)";
-
- this.applyRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive);
- };
-
- /**
- * Add a exception to the exceptions array
- * and replace old with new exception.
- *
- * @param {String} exception RegExp as string
- * @param {Boolean} isActive Is this exception acitve?
- */
- this.addException = function (exception, isActive = true) {
- if (isActive) {
- enabled_exceptions[exception] = true;
-
- if (disabled_exceptions[exception] !== undefined) {
- delete disabled_exceptions[exception];
- }
- } else {
- disabled_exceptions[exception] = true;
-
- if (enabled_exceptions[exception] !== undefined) {
- delete enabled_exceptions[exception];
- }
- }
- };
-
- /**
- * Private helper method to check if the url
- * an exception.
- *
- * @param {String} url RegExp as string
- * @return {boolean} if matching? true: false
- */
- this.matchException = function (url) {
- let result = false;
-
- //Add the site blocked alert to every exception
- if (url === siteBlockedAlert) return true;
-
- for (const exception in enabled_exceptions) {
- if (result) break;
-
- let exception_regex = new RegExp(exception, "i");
- result = exception_regex.test(url);
- }
-
- return result;
- };
-
- /**
- * Add a redirection to the redirections array
- * and replace old with new redirection.
- *
- * @param {String} redirection RegExp as string
- * @param {Boolean} isActive Is this redirection active?
- */
- this.addRedirection = function (redirection, isActive = true) {
- if (isActive) {
- enabled_redirections[redirection] = true;
-
- if (disabled_redirections[redirection] !== undefined) {
- delete disabled_redirections[redirection];
- }
- } else {
- disabled_redirections[redirection] = true;
-
- if (enabled_redirections[redirection] !== undefined) {
- delete enabled_redirections[redirection];
- }
- }
- };
-
- /**
- * Return all redirection.
- *
- * @return url
- */
- this.getRedirection = function (url) {
- let re = null;
-
- for (const redirection in enabled_redirections) {
- let result = (url.match(new RegExp(redirection, "i")));
-
- if (result && result.length > 0 && redirection) {
- re = (new RegExp(redirection, "i")).exec(url)[1];
-
- break;
- }
- }
-
- return re;
- };
- }
-
- // ##################################################################
-
- /**
- * Function which called from the webRequest to
- * remove the tracking fields from the url.
- *
- * @param {webRequest} request webRequest-Object
- * @return {Array} redirectUrl or none
- */
- function clearUrl(request) {
- const URLbeforeReplaceCount = countFields(request.url);
-
- //Add Fields form Request to global url counter
- increaseGlobalURLCounter(URLbeforeReplaceCount);
-
- if (storage.globalStatus) {
- let result = {
- "changes": false,
- "url": "",
- "redirect": false,
- "cancel": false
- };
-
- if (storage.pingBlocking && storage.pingRequestTypes.includes(request.type)) {
- pushToLog(request.url, request.url, translate('log_ping_blocked'));
- increaseBadged(false, request);
- increaseGlobalURLCounter(1);
- return {cancel: true};
- }
-
- /*
- * Call for every provider the removeFieldsFormURL method.
- */
- for (let i = 0; i < providers.length; i++) {
- if (providers[i].matchURL(request.url)) {
- result = removeFieldsFormURL(providers[i], request.url, false, request);
- }
-
- /*
- * Expand urls and bypass tracking.
- * Cancel the active request.
- */
- if (result.redirect) {
- if (providers[i].shouldForceRedirect() &&
- request.type === 'main_frame') {
- browser.tabs.update(request.tabId, {url: result.url}).catch(handleError);
- return {cancel: true};
- }
-
- return {
- redirectUrl: result.url
- };
- }
-
- /*
- * Cancel the Request and redirect to the site blocked alert page,
- * to inform the user about the full url blocking.
- */
- if (result.cancel) {
- if (request.type === 'main_frame') {
- const blockingPage = browser.extension.getURL("html/siteBlockedAlert.html?source=" + encodeURIComponent(request.url));
- browser.tabs.update(request.tabId, {url: blockingPage}).catch(handleError);
-
- return {cancel: true};
- } else {
- return {
- redirectUrl: siteBlockedAlert
- };
- }
- }
-
- /*
- * Ensure that the function go not into
- * a loop.
- */
- if (result.changes) {
- return {
- redirectUrl: result.url
- };
- }
- }
- }
-
- // Default case
- return {};
- }
-
- /**
- * Call loadOldDataFromStore, getHash, counter, status and log functions
- */
-
- loadOldDataFromStore();
- getHash();
- setBadgedStatus();
-
- /**
- * Check the request.
- */
- function promise(requestDetails) {
- if (isDataURL(requestDetails)) {
- return {};
- } else {
- return clearUrl(requestDetails);
- }
- }
-
- /**
- * To prevent long loading on data urls
- * we will check here for data urls.
- *
- * @type {requestDetails}
- * @return {boolean}
- */
- function isDataURL(requestDetails) {
- const s = requestDetails.url;
-
- return s.substring(0, 4) === "data";
- }
-
- /**
- * Call by each Request and checking the url.
- *
- * @type {Array}
- */
- browser.webRequest.onBeforeRequest.addListener(
- promise,
- {urls: [""], types: getData("types").concat(getData("pingRequestTypes"))},
- ["blocking"]
- );
-}
+/*
+* ClearURLs
+* Copyright (c) 2017-2020 Kevin Röbert
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public License
+* along with this program. If not, see .
+*/
+
+/*jshint esversion: 6 */
+/*
+* This script is responsible for the core functionalities.
+*/
+var providers = [];
+var prvKeys = [];
+var siteBlockedAlert = 'javascript:void(0)';
+var dataHash;
+var localDataHash;
+var os;
+
+/**
+ * Helper function which remove the tracking fields
+ * for each provider given as parameter.
+ *
+ * @param {Provider} provider Provider-Object
+ * @param pureUrl URL as String
+ * @param {boolean} quiet if the action should be displayed in log and statistics
+ * @param {requestDetails} request the request details
+ * @return {Array} Array with changes and url fields
+ */
+function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) {
+ let url = pureUrl;
+ let domain = "";
+ let fragments = "";
+ let fields = "";
+ let rules = provider.getRules();
+ let changes = false;
+ let cancel = false;
+ let rawRules = provider.getRawRules();
+
+ if (storage.localHostsSkipping && checkLocalURL(pureUrl)) {
+ return {
+ "changes": false,
+ "url": url,
+ "cancel": false
+ };
+ }
+
+ /*
+ * Apply raw rules to the URL.
+ */
+ rawRules.forEach(function (rawRule) {
+ let beforeReplace = url;
+ url = url.replace(new RegExp(rawRule, "gi"), "");
+
+ if (beforeReplace !== url) {
+ //Log the action
+ if (storage.loggingStatus && !quiet) {
+ pushToLog(beforeReplace, url, rawRule);
+ }
+
+ increaseBadged(quiet, request);
+ changes = true;
+ }
+ });
+
+ if (existsFragments(url)) {
+ domain = url.replace(new RegExp("#.*", "i"), "");
+ }
+ if (existsFields(url)) {
+ domain = url.replace(new RegExp("\\?.*", "i"), "");
+ }
+
+ /*
+ * Expand the url by provider redirections. So no tracking on
+ * url redirections form sites to sites.
+ */
+ let re = provider.getRedirection(url);
+ if (re !== null) {
+ url = decodeURL(re);
+
+ //Log the action
+ if (!quiet) {
+ pushToLog(pureUrl, url, translate('log_redirect'));
+ increaseGlobalURLCounter(1);
+ increaseBadged(false, request)
+ };
+
+ return {
+ "redirect": true,
+ "url": url
+ };
+ }
+
+ if (existsFields(url)) {
+ fields = "?" + extractFileds(url).rmEmpty().join("&");
+ }
+
+ if (existsFragments(url)) {
+ fragments = "#" + extractFragments(url).rmEmpty().join("&");
+ }
+
+ /**
+ * Only test for matches, if there are fields or fragments that can be cleaned.
+ */
+ if (fields !== "" || fragments !== "") {
+ rules.forEach(function (rule) {
+ let beforeReplace = fields;
+ let beforeReplaceFragments = fragments;
+ fields = fields.replace(new RegExp(rule, "gi"), "");
+ fragments = fragments.replace(new RegExp(rule, "gi"), "");
+
+ if (beforeReplace !== fields || beforeReplaceFragments !== fragments) {
+ //Log the action
+ if (storage.loggingStatus) {
+ let tempURL = domain;
+ let tempBeforeURL = domain;
+
+ if (fields !== "") tempURL += "?" + fields.replace("?&", "?").replace("?", "");
+ if (fragments !== "") tempURL += "#" + fragments.replace("#&", "#").replace("#", "");
+ if (beforeReplace !== "") tempBeforeURL += "?" + beforeReplace.replace("?&", "?").replace("?", "");
+ if (beforeReplaceFragments !== "") tempBeforeURL += "#" + beforeReplaceFragments.replace("#&", "#").replace("#", "");
+
+ if (!quiet) pushToLog(tempBeforeURL, tempURL, rule);
+ }
+
+ increaseBadged(quiet, request);
+ changes = true;
+ }
+ });
+
+ let finalURL = domain;
+
+ if (fields !== "") finalURL += "?" + fields.replace("?", "");
+ if (fragments !== "") finalURL += "#" + fragments.replace("#", "");
+
+ url = finalURL.replace(new RegExp("\\?&"), "?").replace(new RegExp("#&"), "#");
+ }
+
+ if (provider.isCaneling() && storage.domainBlocking) {
+ if (!quiet) pushToLog(pureUrl, pureUrl, translate('log_domain_blocked'));
+ increaseGlobalURLCounter(1);
+ increaseBadged(quiet, request);
+ cancel = true;
+ }
+
+ return {
+ "changes": changes,
+ "url": url,
+ "cancel": cancel
+ };
+}
+
+function start() {
+ /**
+ * Initialize the JSON provider object keys.
+ *
+ * @param {object} obj
+ */
+ function getKeys(obj) {
+ for (const key in obj) {
+ prvKeys.push(key);
+ }
+ }
+
+ /**
+ * Initialize the providers form the JSON object.
+ *
+ */
+ function createProviders() {
+ let data = storage.ClearURLsData;
+
+ for (let p = 0; p < prvKeys.length; p++) {
+ //Create new provider
+ providers.push(new Provider(prvKeys[p], data.providers[prvKeys[p]].getOrDefault('completeProvider', false),
+ data.providers[prvKeys[p]].getOrDefault('forceRedirection', false)));
+
+ //Add URL Pattern
+ providers[p].setURLPattern(data.providers[prvKeys[p]].getOrDefault('urlPattern', ''));
+
+ let rules = data.providers[prvKeys[p]].getOrDefault('rules', []);
+ //Add rules to provider
+ for (let r = 0; r < rules.length; r++) {
+ providers[p].addRule(rules[r]);
+ }
+
+ let rawRules = data.providers[prvKeys[p]].getOrDefault('rawRules', []);
+ //Add raw rules to provider
+ for (let raw = 0; raw < rawRules.length; raw++) {
+ providers[p].addRawRule(rawRules[raw]);
+ }
+
+ let referralMarketingRules = data.providers[prvKeys[p]].getOrDefault('referralMarketing', []);
+ //Add referral marketing rules to provider
+ for (let referralMarketing = 0; referralMarketing < referralMarketingRules.length; referralMarketing++) {
+ providers[p].addReferralMarketing(referralMarketingRules[referralMarketing]);
+ }
+
+ let exceptions = data.providers[prvKeys[p]].getOrDefault('exceptions', []);
+ //Add exceptions to provider
+ for (let e = 0; e < exceptions.length; e++) {
+ providers[p].addException(exceptions[e]);
+ }
+
+ let redirections = data.providers[prvKeys[p]].getOrDefault('redirections', []);
+ //Add redirections to provider
+ for (let re = 0; re < redirections.length; re++) {
+ providers[p].addRedirection(redirections[re]);
+ }
+ }
+ }
+
+ /**
+ * Convert the external data to Objects and
+ * call the create provider function.
+ *
+ * @param {String} retrievedText - pure data form github
+ */
+ function toObject(retrievedText) {
+ getKeys(storage.ClearURLsData.providers);
+ createProviders();
+ }
+
+ /**
+ * Deactivates ClearURLs, if no rules can be downloaded and also no old rules in storage
+ */
+ function deactivateOnFailure() {
+ if(storage.ClearURLsData.length === 0) {
+ storage.globalStatus = false;
+ storage.dataHash = "";
+ changeIcon();
+ storeHashStatus(5);
+ saveOnExit();
+ }
+ }
+
+ /**
+ * Get the hash for the rule file on GitLab.
+ * Check the hash with the hash form the local file.
+ * If the hash has changed, then download the new rule file.
+ * Else do nothing.
+ */
+ function getHash() {
+ //Get the target hash from GitLab
+ fetch(storage.hashURL)
+ .then(function (response) {
+ const responseTextHash = response.clone().text().then(function (responseTextHash) {
+ if (response.ok && $.trim(responseTextHash)) {
+ dataHash = responseTextHash;
+
+ if ($.trim(dataHash) !== $.trim(localDataHash)) {
+ fetchFromURL();
+ } else {
+ toObject(storage.ClearURLsData);
+ storeHashStatus(1);
+ saveOnDisk(['hashStatus']);
+ }
+ } else {
+ dataHash = false;
+ deactivateOnFailure();
+ }
+ });
+ });
+ }
+
+ /*
+ * ##################################################################
+ * # Fetch Rules & Exception from URL #
+ * ##################################################################
+ */
+ function fetchFromURL() {
+ fetch(storage.ruleURL)
+ .then(checkResponse);
+
+ function checkResponse(response) {
+ const responseText = response.clone().text().then(function (responseText) {
+ if (response.ok && $.trim(responseText)) {
+ const downloadedFileHash = $.sha256(responseText);
+
+ if ($.trim(downloadedFileHash) === $.trim(dataHash)) {
+ storage.ClearURLsData = responseText;
+ storage.dataHash = downloadedFileHash;
+ storeHashStatus(2);
+ } else {
+ storeHashStatus(3);
+ }
+ storage.ClearURLsData = JSON.parse(storage.ClearURLsData);
+ toObject(storage.ClearURLsData);
+ saveOnDisk(['ClearURLsData', 'dataHash', 'hashStatus']);
+ } else {
+ deactivateOnFailure();
+ }
+ });
+ }
+ }
+
+ // ##################################################################
+
+ /*
+ * ##################################################################
+ * # Supertyp Provider #
+ * ##################################################################
+ */
+ /**
+ * Declare constructor
+ *
+ * @param {String} _name Provider name
+ * @param {boolean} _completeProvider Set URL Pattern as rule
+ * @param {boolean} _forceRedirection Whether redirects should be enforced via a "tabs.update"
+ * @param {boolean} _isActive Is the provider active?
+ */
+ function Provider(_name, _completeProvider = false, _forceRedirection = false, _isActive = true) {
+ let name = _name;
+ let urlPattern;
+ let enabled_rules = {};
+ let disabled_rules = {};
+ let enabled_exceptions = {};
+ let disabled_exceptions = {};
+ let canceling = _completeProvider;
+ let enabled_redirections = {};
+ let disabled_redirections = {};
+ let active = _isActive;
+ let enabled_rawRules = {};
+ let disabled_rawRules = {};
+ let enabled_referralMarketing = {};
+ let disabled_referralMarketing = {};
+
+ if (_completeProvider) {
+ enabled_rules[".*"] = true;
+ }
+
+ /**
+ * Returns whether redirects should be enforced via a "tabs.update"
+ * @return {boolean} whether redirects should be enforced
+ */
+ this.shouldForceRedirect = function () {
+ return _forceRedirection;
+ };
+
+ /**
+ * Returns the provider name.
+ * @return {String}
+ */
+ this.getName = function () {
+ return name;
+ };
+
+ /**
+ * Add URL pattern.
+ *
+ * @require urlPatterns as RegExp
+ */
+ this.setURLPattern = function (urlPatterns) {
+ urlPattern = new RegExp(urlPatterns, "i");
+ };
+
+ /**
+ * Return if the Provider Request is canceled
+ * @return {Boolean} isCanceled
+ */
+ this.isCaneling = function () {
+ return canceling;
+ };
+
+ /**
+ * Check the url is matching the ProviderURL.
+ *
+ * @return {boolean} ProviderURL as RegExp
+ */
+ this.matchURL = function (url) {
+ return urlPattern.test(url) && !(this.matchException(url));
+ };
+
+ /**
+ * Apply a rule to a given tuple of rule array.
+ * @param enabledRuleArray array for enabled rules
+ * @param disabledRulesArray array for disabled rules
+ * @param {String} rule RegExp as string
+ * @param {boolean} isActive Is this rule active?
+ */
+ this.applyRule = (enabledRuleArray, disabledRulesArray, rule, isActive = true) => {
+ if (isActive) {
+ enabledRuleArray[rule] = true;
+
+ if (disabledRulesArray[rule] !== undefined) {
+ delete disabledRulesArray[rule];
+ }
+ } else {
+ disabledRulesArray[rule] = true;
+
+ if (enabledRuleArray[rule] !== undefined) {
+ delete enabledRuleArray[rule];
+ }
+ }
+ };
+
+ /**
+ * Add a rule to the rule array
+ * and replace old rule with new rule.
+ *
+ * @param {String} rule RegExp as string
+ * @param {boolean} isActive Is this rule active?
+ */
+ this.addRule = function (rule, isActive = true) {
+ rule = "([\\/\\?#]|(&|&))+(" + rule + "=[^&]*)";
+
+ this.applyRule(enabled_rules, disabled_rules, rule, isActive);
+ };
+
+ /**
+ * Return all active rules as an array.
+ *
+ * @return Array RegExp strings
+ */
+ this.getRules = function () {
+ if (!storage.referralMarketing) {
+ return Object.keys(Object.assign(enabled_rules, enabled_referralMarketing));
+ }
+
+ return Object.keys(enabled_rules);
+ };
+
+ /**
+ * Add a raw rule to the raw rule array
+ * and replace old raw rule with new raw rule.
+ *
+ * @param {String} rule RegExp as string
+ * @param {boolean} isActive Is this rule active?
+ */
+ this.addRawRule = function (rule, isActive = true) {
+ this.applyRule(enabled_rawRules, disabled_rawRules, rule, isActive);
+ };
+
+ /**
+ * Return all active raw rules as an array.
+ *
+ * @return Array RegExp strings
+ */
+ this.getRawRules = function () {
+ return Object.keys(enabled_rawRules);
+ };
+
+ /**
+ * Add a referral marketing rule to the referral marketing array
+ * and replace old referral marketing rule with new referral marketing rule.
+ *
+ * @param {String} rule RegExp as string
+ * @param {boolean} isActive Is this rule active?
+ */
+ this.addReferralMarketing = function (rule, isActive = true) {
+ rule = "([\\/\\?#]|(&|&))+(" + rule + "=[^\\/\\?&]*)";
+
+ this.applyRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive);
+ };
+
+ /**
+ * Add a exception to the exceptions array
+ * and replace old with new exception.
+ *
+ * @param {String} exception RegExp as string
+ * @param {Boolean} isActive Is this exception acitve?
+ */
+ this.addException = function (exception, isActive = true) {
+ if (isActive) {
+ enabled_exceptions[exception] = true;
+
+ if (disabled_exceptions[exception] !== undefined) {
+ delete disabled_exceptions[exception];
+ }
+ } else {
+ disabled_exceptions[exception] = true;
+
+ if (enabled_exceptions[exception] !== undefined) {
+ delete enabled_exceptions[exception];
+ }
+ }
+ };
+
+ /**
+ * Private helper method to check if the url
+ * an exception.
+ *
+ * @param {String} url RegExp as string
+ * @return {boolean} if matching? true: false
+ */
+ this.matchException = function (url) {
+ let result = false;
+
+ //Add the site blocked alert to every exception
+ if (url === siteBlockedAlert) return true;
+
+ for (const exception in enabled_exceptions) {
+ if (result) break;
+
+ let exception_regex = new RegExp(exception, "i");
+ result = exception_regex.test(url);
+ }
+
+ return result;
+ };
+
+ /**
+ * Add a redirection to the redirections array
+ * and replace old with new redirection.
+ *
+ * @param {String} redirection RegExp as string
+ * @param {Boolean} isActive Is this redirection active?
+ */
+ this.addRedirection = function (redirection, isActive = true) {
+ if (isActive) {
+ enabled_redirections[redirection] = true;
+
+ if (disabled_redirections[redirection] !== undefined) {
+ delete disabled_redirections[redirection];
+ }
+ } else {
+ disabled_redirections[redirection] = true;
+
+ if (enabled_redirections[redirection] !== undefined) {
+ delete enabled_redirections[redirection];
+ }
+ }
+ };
+
+ /**
+ * Return all redirection.
+ *
+ * @return url
+ */
+ this.getRedirection = function (url) {
+ let re = null;
+
+ for (const redirection in enabled_redirections) {
+ let result = (url.match(new RegExp(redirection, "i")));
+
+ if (result && result.length > 0 && redirection) {
+ re = (new RegExp(redirection, "i")).exec(url)[1];
+
+ break;
+ }
+ }
+
+ return re;
+ };
+ }
+
+ // ##################################################################
+
+ /**
+ * Function which called from the webRequest to
+ * remove the tracking fields from the url.
+ *
+ * @param {webRequest} request webRequest-Object
+ * @return {Array} redirectUrl or none
+ */
+ function clearUrl(request) {
+ const URLbeforeReplaceCount = countFields(request.url);
+
+ //Add Fields form Request to global url counter
+ increaseGlobalURLCounter(URLbeforeReplaceCount);
+
+ if (storage.globalStatus) {
+ let result = {
+ "changes": false,
+ "url": "",
+ "redirect": false,
+ "cancel": false
+ };
+
+ if (storage.pingBlocking && storage.pingRequestTypes.includes(request.type)) {
+ pushToLog(request.url, request.url, translate('log_ping_blocked'));
+ increaseBadged(false, request);
+ increaseGlobalURLCounter(1);
+ return {cancel: true};
+ }
+
+ /*
+ * Call for every provider the removeFieldsFormURL method.
+ */
+ for (let i = 0; i < providers.length; i++) {
+ if (providers[i].matchURL(request.url)) {
+ result = removeFieldsFormURL(providers[i], request.url, false, request);
+ }
+
+ /*
+ * Expand urls and bypass tracking.
+ * Cancel the active request.
+ */
+ if (result.redirect) {
+ if (providers[i].shouldForceRedirect() &&
+ request.type === 'main_frame') {
+ browser.tabs.update(request.tabId, {url: result.url}).catch(handleError);
+ return {cancel: true};
+ }
+
+ return {
+ redirectUrl: result.url
+ };
+ }
+
+ /*
+ * Cancel the Request and redirect to the site blocked alert page,
+ * to inform the user about the full url blocking.
+ */
+ if (result.cancel) {
+ if (request.type === 'main_frame') {
+ const blockingPage = browser.extension.getURL("html/siteBlockedAlert.html?source=" + encodeURIComponent(request.url));
+ browser.tabs.update(request.tabId, {url: blockingPage}).catch(handleError);
+
+ return {cancel: true};
+ } else {
+ return {
+ redirectUrl: siteBlockedAlert
+ };
+ }
+ }
+
+ /*
+ * Ensure that the function go not into
+ * a loop.
+ */
+ if (result.changes) {
+ return {
+ redirectUrl: result.url
+ };
+ }
+ }
+ }
+
+ // Default case
+ return {};
+ }
+
+ /**
+ * Call loadOldDataFromStore, getHash, counter, status and log functions
+ */
+
+ loadOldDataFromStore();
+ getHash();
+ setBadgedStatus();
+
+ /**
+ * Check the request.
+ */
+ function promise(requestDetails) {
+ if (isDataURL(requestDetails)) {
+ return {};
+ } else {
+ return clearUrl(requestDetails);
+ }
+ }
+
+ /**
+ * To prevent long loading on data urls
+ * we will check here for data urls.
+ *
+ * @type {requestDetails}
+ * @return {boolean}
+ */
+ function isDataURL(requestDetails) {
+ const s = requestDetails.url;
+
+ return s.substring(0, 4) === "data";
+ }
+
+ /**
+ * Call by each Request and checking the url.
+ *
+ * @type {Array}
+ */
+ browser.webRequest.onBeforeRequest.addListener(
+ promise,
+ {urls: [""], types: getData("types").concat(getData("pingRequestTypes"))},
+ ["blocking"]
+ );
+}
diff --git a/data/data.min.json b/data/data.min.json
index 7fc92fb..50b04df 100644
--- a/data/data.min.json
+++ b/data/data.min.json
@@ -135,7 +135,8 @@
"urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-.]*\\.)?(google)(\\.[a-zA-Z]{2,})\\/search\\?.*",
"completeProvider": false,
"rules": [
- "client"
+ "client",
+ "sclient"
],
"referralMarketing": [],
"rawRules": [],
@@ -403,7 +404,18 @@
"reddit": {
"urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-.]*\\.)?(reddit)(\\.[a-zA-Z]{2,}).*",
"completeProvider": false,
- "rules": [],
+ "rules": [
+ "%24deep_link",
+ "\\$deep_link",
+ "correlation_id",
+ "ref_campaign",
+ "ref_source",
+ "%243p",
+ "\\$3p",
+ "%24original_url",
+ "\\$original_url",
+ "_branch_match_id"
+ ],
"referralMarketing": [],
"rawRules": [],
"exceptions": [],
@@ -1667,6 +1679,35 @@
"exceptions": [],
"redirections": [],
"forceRedirection": false
+ },
+ "bestbuy.com": {
+ "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-.]*\\.)?(bestbuy)(\\.com).*",
+ "completeProvider": false,
+ "rules": [
+ "irclickid",
+ "irgwc",
+ "loc",
+ "acampID",
+ "mpid",
+ "intl"
+ ],
+ "referralMarketing": [],
+ "rawRules": [],
+ "exceptions": [],
+ "redirections": [],
+ "forceRedirection": false
+ },
+ "digidip.net": {
+ "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-.]*\\.)?(digidip)(\\.net).*",
+ "completeProvider": false,
+ "rules": [],
+ "referralMarketing": [],
+ "rawRules": [],
+ "exceptions": [],
+ "redirections": [
+ ".*url=([^&]*)"
+ ],
+ "forceRedirection": false
}
}
}