From 201f3a9edb0620c25862ee7b61689656bf73a0fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20R=C3=B6bert?= Date: Fri, 24 Apr 2020 01:17:58 +0200 Subject: [PATCH] Updated rules #463 #468 #469 #470 --- clearurls.js | 1370 ++++++++++++++++++++++---------------------- data/data.min.json | 45 +- 2 files changed, 728 insertions(+), 687 deletions(-) diff --git a/clearurls.js b/clearurls.js index bf4d388..6f1f6ea 100644 --- a/clearurls.js +++ b/clearurls.js @@ -1,685 +1,685 @@ -/* -* ClearURLs -* Copyright (c) 2017-2020 Kevin Röbert -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Lesser General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser General Public License for more details. -* -* You should have received a copy of the GNU Lesser General Public License -* along with this program. If not, see . -*/ - -/*jshint esversion: 6 */ -/* -* This script is responsible for the core functionalities. -*/ -var providers = []; -var prvKeys = []; -var siteBlockedAlert = 'javascript:void(0)'; -var dataHash; -var localDataHash; -var os; - -/** - * Helper function which remove the tracking fields - * for each provider given as parameter. - * - * @param {Provider} provider Provider-Object - * @param pureUrl URL as String - * @param {boolean} quiet if the action should be displayed in log and statistics - * @param {requestDetails} request the request details - * @return {Array} Array with changes and url fields - */ -function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) { - let url = pureUrl; - let domain = ""; - let fragments = ""; - let fields = ""; - let rules = provider.getRules(); - let changes = false; - let cancel = false; - let rawRules = provider.getRawRules(); - - if (storage.localHostsSkipping && checkLocalURL(pureUrl)) { - return { - "changes": false, - "url": url, - "cancel": false - }; - } - - /* - * Apply raw rules to the URL. - */ - rawRules.forEach(function (rawRule) { - let beforeReplace = url; - url = url.replace(new RegExp(rawRule, "gi"), ""); - - if (beforeReplace !== url) { - //Log the action - if (storage.loggingStatus && !quiet) { - pushToLog(beforeReplace, url, rawRule); - } - - increaseBadged(quiet, request); - changes = true; - } - }); - - if (existsFragments(url)) { - domain = url.replace(new RegExp("#.*", "i"), ""); - } - if (existsFields(url)) { - domain = url.replace(new RegExp("\\?.*", "i"), ""); - } - - /* - * Expand the url by provider redirections. So no tracking on - * url redirections form sites to sites. - */ - let re = provider.getRedirection(url); - if (re !== null) { - url = decodeURL(re); - - //Log the action - if (!quiet) { - pushToLog(pureUrl, url, translate('log_redirect')); - increaseGlobalURLCounter(1); - increaseBadged(false, request) - }; - - return { - "redirect": true, - "url": url - }; - } - - if (existsFields(url)) { - fields = "?" + extractFileds(url).rmEmpty().join("&"); - } - - if (existsFragments(url)) { - fragments = "#" + extractFragments(url).rmEmpty().join("&"); - } - - /** - * Only test for matches, if there are fields or fragments that can be cleaned. - */ - if (fields !== "" || fragments !== "") { - rules.forEach(function (rule) { - let beforeReplace = fields; - let beforeReplaceFragments = fragments; - fields = fields.replace(new RegExp(rule, "gi"), ""); - fragments = fragments.replace(new RegExp(rule, "gi"), ""); - - if (beforeReplace !== fields || beforeReplaceFragments !== fragments) { - //Log the action - if (storage.loggingStatus) { - let tempURL = domain; - let tempBeforeURL = domain; - - if (fields !== "") tempURL += "?" + fields.replace("?&", "?").replace("?", ""); - if (fragments !== "") tempURL += "#" + fragments.replace("#&", "#").replace("#", ""); - if (beforeReplace !== "") tempBeforeURL += "?" + beforeReplace.replace("?&", "?").replace("?", ""); - if (beforeReplaceFragments !== "") tempBeforeURL += "#" + beforeReplaceFragments.replace("#&", "#").replace("#", ""); - - if (!quiet) pushToLog(tempBeforeURL, tempURL, rule); - } - - increaseBadged(quiet, request); - changes = true; - } - }); - - let finalURL = domain; - - if (fields !== "") finalURL += "?" + fields.replace("?", ""); - if (fragments !== "") finalURL += "#" + fragments.replace("#", ""); - - url = finalURL.replace(new RegExp("\\?&"), "?").replace(new RegExp("#&"), "#"); - } - - if (provider.isCaneling() && storage.domainBlocking) { - if (!quiet) pushToLog(pureUrl, pureUrl, translate('log_domain_blocked')); - increaseGlobalURLCounter(1); - increaseBadged(quiet, request); - cancel = true; - } - - return { - "changes": changes, - "url": url, - "cancel": cancel - }; -} - -function start() { - /** - * Initialize the JSON provider object keys. - * - * @param {object} obj - */ - function getKeys(obj) { - for (const key in obj) { - prvKeys.push(key); - } - } - - /** - * Initialize the providers form the JSON object. - * - */ - function createProviders() { - let data = storage.ClearURLsData; - - for (let p = 0; p < prvKeys.length; p++) { - //Create new provider - providers.push(new Provider(prvKeys[p], data.providers[prvKeys[p]].getOrDefault('completeProvider', false), - data.providers[prvKeys[p]].getOrDefault('forceRedirection', false))); - - //Add URL Pattern - providers[p].setURLPattern(data.providers[prvKeys[p]].getOrDefault('urlPattern', '')); - - let rules = data.providers[prvKeys[p]].getOrDefault('rules', []); - //Add rules to provider - for (let r = 0; r < rules.length; r++) { - providers[p].addRule(rules[r]); - } - - let rawRules = data.providers[prvKeys[p]].getOrDefault('rawRules', []); - //Add raw rules to provider - for (let raw = 0; raw < rawRules.length; raw++) { - providers[p].addRawRule(rawRules[raw]); - } - - let referralMarketingRules = data.providers[prvKeys[p]].getOrDefault('referralMarketing', []); - //Add referral marketing rules to provider - for (let referralMarketing = 0; referralMarketing < referralMarketingRules.length; referralMarketing++) { - providers[p].addReferralMarketing(referralMarketingRules[referralMarketing]); - } - - let exceptions = data.providers[prvKeys[p]].getOrDefault('exceptions', []); - //Add exceptions to provider - for (let e = 0; e < exceptions.length; e++) { - providers[p].addException(exceptions[e]); - } - - let redirections = data.providers[prvKeys[p]].getOrDefault('redirections', []); - //Add redirections to provider - for (let re = 0; re < redirections.length; re++) { - providers[p].addRedirection(redirections[re]); - } - } - } - - /** - * Convert the external data to Objects and - * call the create provider function. - * - * @param {String} retrievedText - pure data form github - */ - function toObject(retrievedText) { - getKeys(storage.ClearURLsData.providers); - createProviders(); - } - - /** - * Deactivates ClearURLs, if no rules can be downloaded and also no old rules in storage - */ - function deactivateOnFailure() { - if(storage.ClearURLsData.length === 0) { - storage.globalStatus = false; - storage.dataHash = ""; - changeIcon(); - storeHashStatus(5); - saveOnExit(); - } - } - - /** - * Get the hash for the rule file on GitLab. - * Check the hash with the hash form the local file. - * If the hash has changed, then download the new rule file. - * Else do nothing. - */ - function getHash() { - //Get the target hash from GitLab - fetch(storage.hashURL) - .then(function (response) { - const responseTextHash = response.clone().text().then(function (responseTextHash) { - if (response.ok && $.trim(responseTextHash)) { - dataHash = responseTextHash; - - if ($.trim(dataHash) !== $.trim(localDataHash)) { - fetchFromURL(); - } else { - toObject(storage.ClearURLsData); - storeHashStatus(1); - saveOnDisk(['hashStatus']); - } - } else { - dataHash = false; - deactivateOnFailure(); - } - }); - }); - } - - /* - * ################################################################## - * # Fetch Rules & Exception from URL # - * ################################################################## - */ - function fetchFromURL() { - fetch(storage.ruleURL) - .then(checkResponse); - - function checkResponse(response) { - const responseText = response.clone().text().then(function (responseText) { - if (response.ok && $.trim(responseText)) { - const downloadedFileHash = $.sha256(responseText); - - if ($.trim(downloadedFileHash) === $.trim(dataHash)) { - storage.ClearURLsData = responseText; - storage.dataHash = downloadedFileHash; - storeHashStatus(2); - } else { - storeHashStatus(3); - } - storage.ClearURLsData = JSON.parse(storage.ClearURLsData); - toObject(storage.ClearURLsData); - saveOnDisk(['ClearURLsData', 'dataHash', 'hashStatus']); - } else { - deactivateOnFailure(); - } - }); - } - } - - // ################################################################## - - /* - * ################################################################## - * # Supertyp Provider # - * ################################################################## - */ - /** - * Declare constructor - * - * @param {String} _name Provider name - * @param {boolean} _completeProvider Set URL Pattern as rule - * @param {boolean} _forceRedirection Whether redirects should be enforced via a "tabs.update" - * @param {boolean} _isActive Is the provider active? - */ - function Provider(_name, _completeProvider = false, _forceRedirection = false, _isActive = true) { - let name = _name; - let urlPattern; - let enabled_rules = {}; - let disabled_rules = {}; - let enabled_exceptions = {}; - let disabled_exceptions = {}; - let canceling = _completeProvider; - let enabled_redirections = {}; - let disabled_redirections = {}; - let active = _isActive; - let enabled_rawRules = {}; - let disabled_rawRules = {}; - let enabled_referralMarketing = {}; - let disabled_referralMarketing = {}; - - if (_completeProvider) { - enabled_rules[".*"] = true; - } - - /** - * Returns whether redirects should be enforced via a "tabs.update" - * @return {boolean} whether redirects should be enforced - */ - this.shouldForceRedirect = function () { - return _forceRedirection; - }; - - /** - * Returns the provider name. - * @return {String} - */ - this.getName = function () { - return name; - }; - - /** - * Add URL pattern. - * - * @require urlPatterns as RegExp - */ - this.setURLPattern = function (urlPatterns) { - urlPattern = new RegExp(urlPatterns, "i"); - }; - - /** - * Return if the Provider Request is canceled - * @return {Boolean} isCanceled - */ - this.isCaneling = function () { - return canceling; - }; - - /** - * Check the url is matching the ProviderURL. - * - * @return {boolean} ProviderURL as RegExp - */ - this.matchURL = function (url) { - return urlPattern.test(url) && !(this.matchException(url)); - }; - - /** - * Apply a rule to a given tuple of rule array. - * @param enabledRuleArray array for enabled rules - * @param disabledRulesArray array for disabled rules - * @param {String} rule RegExp as string - * @param {boolean} isActive Is this rule active? - */ - this.applyRule = (enabledRuleArray, disabledRulesArray, rule, isActive = true) => { - if (isActive) { - enabledRuleArray[rule] = true; - - if (disabledRulesArray[rule] !== undefined) { - delete disabledRulesArray[rule]; - } - } else { - disabledRulesArray[rule] = true; - - if (enabledRuleArray[rule] !== undefined) { - delete enabledRuleArray[rule]; - } - } - }; - - /** - * Add a rule to the rule array - * and replace old rule with new rule. - * - * @param {String} rule RegExp as string - * @param {boolean} isActive Is this rule active? - */ - this.addRule = function (rule, isActive = true) { - rule = "([\\/\\?#]|(&|&))+(" + rule + "=[^&]*)"; - - this.applyRule(enabled_rules, disabled_rules, rule, isActive); - }; - - /** - * Return all active rules as an array. - * - * @return Array RegExp strings - */ - this.getRules = function () { - if (!storage.referralMarketing) { - return Object.keys(Object.assign(enabled_rules, enabled_referralMarketing)); - } - - return Object.keys(enabled_rules); - }; - - /** - * Add a raw rule to the raw rule array - * and replace old raw rule with new raw rule. - * - * @param {String} rule RegExp as string - * @param {boolean} isActive Is this rule active? - */ - this.addRawRule = function (rule, isActive = true) { - this.applyRule(enabled_rawRules, disabled_rawRules, rule, isActive); - }; - - /** - * Return all active raw rules as an array. - * - * @return Array RegExp strings - */ - this.getRawRules = function () { - return Object.keys(enabled_rawRules); - }; - - /** - * Add a referral marketing rule to the referral marketing array - * and replace old referral marketing rule with new referral marketing rule. - * - * @param {String} rule RegExp as string - * @param {boolean} isActive Is this rule active? - */ - this.addReferralMarketing = function (rule, isActive = true) { - rule = "([\\/\\?#]|(&|&))+(" + rule + "=[^\\/\\?&]*)"; - - this.applyRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive); - }; - - /** - * Add a exception to the exceptions array - * and replace old with new exception. - * - * @param {String} exception RegExp as string - * @param {Boolean} isActive Is this exception acitve? - */ - this.addException = function (exception, isActive = true) { - if (isActive) { - enabled_exceptions[exception] = true; - - if (disabled_exceptions[exception] !== undefined) { - delete disabled_exceptions[exception]; - } - } else { - disabled_exceptions[exception] = true; - - if (enabled_exceptions[exception] !== undefined) { - delete enabled_exceptions[exception]; - } - } - }; - - /** - * Private helper method to check if the url - * an exception. - * - * @param {String} url RegExp as string - * @return {boolean} if matching? true: false - */ - this.matchException = function (url) { - let result = false; - - //Add the site blocked alert to every exception - if (url === siteBlockedAlert) return true; - - for (const exception in enabled_exceptions) { - if (result) break; - - let exception_regex = new RegExp(exception, "i"); - result = exception_regex.test(url); - } - - return result; - }; - - /** - * Add a redirection to the redirections array - * and replace old with new redirection. - * - * @param {String} redirection RegExp as string - * @param {Boolean} isActive Is this redirection active? - */ - this.addRedirection = function (redirection, isActive = true) { - if (isActive) { - enabled_redirections[redirection] = true; - - if (disabled_redirections[redirection] !== undefined) { - delete disabled_redirections[redirection]; - } - } else { - disabled_redirections[redirection] = true; - - if (enabled_redirections[redirection] !== undefined) { - delete enabled_redirections[redirection]; - } - } - }; - - /** - * Return all redirection. - * - * @return url - */ - this.getRedirection = function (url) { - let re = null; - - for (const redirection in enabled_redirections) { - let result = (url.match(new RegExp(redirection, "i"))); - - if (result && result.length > 0 && redirection) { - re = (new RegExp(redirection, "i")).exec(url)[1]; - - break; - } - } - - return re; - }; - } - - // ################################################################## - - /** - * Function which called from the webRequest to - * remove the tracking fields from the url. - * - * @param {webRequest} request webRequest-Object - * @return {Array} redirectUrl or none - */ - function clearUrl(request) { - const URLbeforeReplaceCount = countFields(request.url); - - //Add Fields form Request to global url counter - increaseGlobalURLCounter(URLbeforeReplaceCount); - - if (storage.globalStatus) { - let result = { - "changes": false, - "url": "", - "redirect": false, - "cancel": false - }; - - if (storage.pingBlocking && storage.pingRequestTypes.includes(request.type)) { - pushToLog(request.url, request.url, translate('log_ping_blocked')); - increaseBadged(false, request); - increaseGlobalURLCounter(1); - return {cancel: true}; - } - - /* - * Call for every provider the removeFieldsFormURL method. - */ - for (let i = 0; i < providers.length; i++) { - if (providers[i].matchURL(request.url)) { - result = removeFieldsFormURL(providers[i], request.url, false, request); - } - - /* - * Expand urls and bypass tracking. - * Cancel the active request. - */ - if (result.redirect) { - if (providers[i].shouldForceRedirect() && - request.type === 'main_frame') { - browser.tabs.update(request.tabId, {url: result.url}).catch(handleError); - return {cancel: true}; - } - - return { - redirectUrl: result.url - }; - } - - /* - * Cancel the Request and redirect to the site blocked alert page, - * to inform the user about the full url blocking. - */ - if (result.cancel) { - if (request.type === 'main_frame') { - const blockingPage = browser.extension.getURL("html/siteBlockedAlert.html?source=" + encodeURIComponent(request.url)); - browser.tabs.update(request.tabId, {url: blockingPage}).catch(handleError); - - return {cancel: true}; - } else { - return { - redirectUrl: siteBlockedAlert - }; - } - } - - /* - * Ensure that the function go not into - * a loop. - */ - if (result.changes) { - return { - redirectUrl: result.url - }; - } - } - } - - // Default case - return {}; - } - - /** - * Call loadOldDataFromStore, getHash, counter, status and log functions - */ - - loadOldDataFromStore(); - getHash(); - setBadgedStatus(); - - /** - * Check the request. - */ - function promise(requestDetails) { - if (isDataURL(requestDetails)) { - return {}; - } else { - return clearUrl(requestDetails); - } - } - - /** - * To prevent long loading on data urls - * we will check here for data urls. - * - * @type {requestDetails} - * @return {boolean} - */ - function isDataURL(requestDetails) { - const s = requestDetails.url; - - return s.substring(0, 4) === "data"; - } - - /** - * Call by each Request and checking the url. - * - * @type {Array} - */ - browser.webRequest.onBeforeRequest.addListener( - promise, - {urls: [""], types: getData("types").concat(getData("pingRequestTypes"))}, - ["blocking"] - ); -} +/* +* ClearURLs +* Copyright (c) 2017-2020 Kevin Röbert +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Lesser General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public License +* along with this program. If not, see . +*/ + +/*jshint esversion: 6 */ +/* +* This script is responsible for the core functionalities. +*/ +var providers = []; +var prvKeys = []; +var siteBlockedAlert = 'javascript:void(0)'; +var dataHash; +var localDataHash; +var os; + +/** + * Helper function which remove the tracking fields + * for each provider given as parameter. + * + * @param {Provider} provider Provider-Object + * @param pureUrl URL as String + * @param {boolean} quiet if the action should be displayed in log and statistics + * @param {requestDetails} request the request details + * @return {Array} Array with changes and url fields + */ +function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) { + let url = pureUrl; + let domain = ""; + let fragments = ""; + let fields = ""; + let rules = provider.getRules(); + let changes = false; + let cancel = false; + let rawRules = provider.getRawRules(); + + if (storage.localHostsSkipping && checkLocalURL(pureUrl)) { + return { + "changes": false, + "url": url, + "cancel": false + }; + } + + /* + * Apply raw rules to the URL. + */ + rawRules.forEach(function (rawRule) { + let beforeReplace = url; + url = url.replace(new RegExp(rawRule, "gi"), ""); + + if (beforeReplace !== url) { + //Log the action + if (storage.loggingStatus && !quiet) { + pushToLog(beforeReplace, url, rawRule); + } + + increaseBadged(quiet, request); + changes = true; + } + }); + + if (existsFragments(url)) { + domain = url.replace(new RegExp("#.*", "i"), ""); + } + if (existsFields(url)) { + domain = url.replace(new RegExp("\\?.*", "i"), ""); + } + + /* + * Expand the url by provider redirections. So no tracking on + * url redirections form sites to sites. + */ + let re = provider.getRedirection(url); + if (re !== null) { + url = decodeURL(re); + + //Log the action + if (!quiet) { + pushToLog(pureUrl, url, translate('log_redirect')); + increaseGlobalURLCounter(1); + increaseBadged(false, request) + }; + + return { + "redirect": true, + "url": url + }; + } + + if (existsFields(url)) { + fields = "?" + extractFileds(url).rmEmpty().join("&"); + } + + if (existsFragments(url)) { + fragments = "#" + extractFragments(url).rmEmpty().join("&"); + } + + /** + * Only test for matches, if there are fields or fragments that can be cleaned. + */ + if (fields !== "" || fragments !== "") { + rules.forEach(function (rule) { + let beforeReplace = fields; + let beforeReplaceFragments = fragments; + fields = fields.replace(new RegExp(rule, "gi"), ""); + fragments = fragments.replace(new RegExp(rule, "gi"), ""); + + if (beforeReplace !== fields || beforeReplaceFragments !== fragments) { + //Log the action + if (storage.loggingStatus) { + let tempURL = domain; + let tempBeforeURL = domain; + + if (fields !== "") tempURL += "?" + fields.replace("?&", "?").replace("?", ""); + if (fragments !== "") tempURL += "#" + fragments.replace("#&", "#").replace("#", ""); + if (beforeReplace !== "") tempBeforeURL += "?" + beforeReplace.replace("?&", "?").replace("?", ""); + if (beforeReplaceFragments !== "") tempBeforeURL += "#" + beforeReplaceFragments.replace("#&", "#").replace("#", ""); + + if (!quiet) pushToLog(tempBeforeURL, tempURL, rule); + } + + increaseBadged(quiet, request); + changes = true; + } + }); + + let finalURL = domain; + + if (fields !== "") finalURL += "?" + fields.replace("?", ""); + if (fragments !== "") finalURL += "#" + fragments.replace("#", ""); + + url = finalURL.replace(new RegExp("\\?&"), "?").replace(new RegExp("#&"), "#"); + } + + if (provider.isCaneling() && storage.domainBlocking) { + if (!quiet) pushToLog(pureUrl, pureUrl, translate('log_domain_blocked')); + increaseGlobalURLCounter(1); + increaseBadged(quiet, request); + cancel = true; + } + + return { + "changes": changes, + "url": url, + "cancel": cancel + }; +} + +function start() { + /** + * Initialize the JSON provider object keys. + * + * @param {object} obj + */ + function getKeys(obj) { + for (const key in obj) { + prvKeys.push(key); + } + } + + /** + * Initialize the providers form the JSON object. + * + */ + function createProviders() { + let data = storage.ClearURLsData; + + for (let p = 0; p < prvKeys.length; p++) { + //Create new provider + providers.push(new Provider(prvKeys[p], data.providers[prvKeys[p]].getOrDefault('completeProvider', false), + data.providers[prvKeys[p]].getOrDefault('forceRedirection', false))); + + //Add URL Pattern + providers[p].setURLPattern(data.providers[prvKeys[p]].getOrDefault('urlPattern', '')); + + let rules = data.providers[prvKeys[p]].getOrDefault('rules', []); + //Add rules to provider + for (let r = 0; r < rules.length; r++) { + providers[p].addRule(rules[r]); + } + + let rawRules = data.providers[prvKeys[p]].getOrDefault('rawRules', []); + //Add raw rules to provider + for (let raw = 0; raw < rawRules.length; raw++) { + providers[p].addRawRule(rawRules[raw]); + } + + let referralMarketingRules = data.providers[prvKeys[p]].getOrDefault('referralMarketing', []); + //Add referral marketing rules to provider + for (let referralMarketing = 0; referralMarketing < referralMarketingRules.length; referralMarketing++) { + providers[p].addReferralMarketing(referralMarketingRules[referralMarketing]); + } + + let exceptions = data.providers[prvKeys[p]].getOrDefault('exceptions', []); + //Add exceptions to provider + for (let e = 0; e < exceptions.length; e++) { + providers[p].addException(exceptions[e]); + } + + let redirections = data.providers[prvKeys[p]].getOrDefault('redirections', []); + //Add redirections to provider + for (let re = 0; re < redirections.length; re++) { + providers[p].addRedirection(redirections[re]); + } + } + } + + /** + * Convert the external data to Objects and + * call the create provider function. + * + * @param {String} retrievedText - pure data form github + */ + function toObject(retrievedText) { + getKeys(storage.ClearURLsData.providers); + createProviders(); + } + + /** + * Deactivates ClearURLs, if no rules can be downloaded and also no old rules in storage + */ + function deactivateOnFailure() { + if(storage.ClearURLsData.length === 0) { + storage.globalStatus = false; + storage.dataHash = ""; + changeIcon(); + storeHashStatus(5); + saveOnExit(); + } + } + + /** + * Get the hash for the rule file on GitLab. + * Check the hash with the hash form the local file. + * If the hash has changed, then download the new rule file. + * Else do nothing. + */ + function getHash() { + //Get the target hash from GitLab + fetch(storage.hashURL) + .then(function (response) { + const responseTextHash = response.clone().text().then(function (responseTextHash) { + if (response.ok && $.trim(responseTextHash)) { + dataHash = responseTextHash; + + if ($.trim(dataHash) !== $.trim(localDataHash)) { + fetchFromURL(); + } else { + toObject(storage.ClearURLsData); + storeHashStatus(1); + saveOnDisk(['hashStatus']); + } + } else { + dataHash = false; + deactivateOnFailure(); + } + }); + }); + } + + /* + * ################################################################## + * # Fetch Rules & Exception from URL # + * ################################################################## + */ + function fetchFromURL() { + fetch(storage.ruleURL) + .then(checkResponse); + + function checkResponse(response) { + const responseText = response.clone().text().then(function (responseText) { + if (response.ok && $.trim(responseText)) { + const downloadedFileHash = $.sha256(responseText); + + if ($.trim(downloadedFileHash) === $.trim(dataHash)) { + storage.ClearURLsData = responseText; + storage.dataHash = downloadedFileHash; + storeHashStatus(2); + } else { + storeHashStatus(3); + } + storage.ClearURLsData = JSON.parse(storage.ClearURLsData); + toObject(storage.ClearURLsData); + saveOnDisk(['ClearURLsData', 'dataHash', 'hashStatus']); + } else { + deactivateOnFailure(); + } + }); + } + } + + // ################################################################## + + /* + * ################################################################## + * # Supertyp Provider # + * ################################################################## + */ + /** + * Declare constructor + * + * @param {String} _name Provider name + * @param {boolean} _completeProvider Set URL Pattern as rule + * @param {boolean} _forceRedirection Whether redirects should be enforced via a "tabs.update" + * @param {boolean} _isActive Is the provider active? + */ + function Provider(_name, _completeProvider = false, _forceRedirection = false, _isActive = true) { + let name = _name; + let urlPattern; + let enabled_rules = {}; + let disabled_rules = {}; + let enabled_exceptions = {}; + let disabled_exceptions = {}; + let canceling = _completeProvider; + let enabled_redirections = {}; + let disabled_redirections = {}; + let active = _isActive; + let enabled_rawRules = {}; + let disabled_rawRules = {}; + let enabled_referralMarketing = {}; + let disabled_referralMarketing = {}; + + if (_completeProvider) { + enabled_rules[".*"] = true; + } + + /** + * Returns whether redirects should be enforced via a "tabs.update" + * @return {boolean} whether redirects should be enforced + */ + this.shouldForceRedirect = function () { + return _forceRedirection; + }; + + /** + * Returns the provider name. + * @return {String} + */ + this.getName = function () { + return name; + }; + + /** + * Add URL pattern. + * + * @require urlPatterns as RegExp + */ + this.setURLPattern = function (urlPatterns) { + urlPattern = new RegExp(urlPatterns, "i"); + }; + + /** + * Return if the Provider Request is canceled + * @return {Boolean} isCanceled + */ + this.isCaneling = function () { + return canceling; + }; + + /** + * Check the url is matching the ProviderURL. + * + * @return {boolean} ProviderURL as RegExp + */ + this.matchURL = function (url) { + return urlPattern.test(url) && !(this.matchException(url)); + }; + + /** + * Apply a rule to a given tuple of rule array. + * @param enabledRuleArray array for enabled rules + * @param disabledRulesArray array for disabled rules + * @param {String} rule RegExp as string + * @param {boolean} isActive Is this rule active? + */ + this.applyRule = (enabledRuleArray, disabledRulesArray, rule, isActive = true) => { + if (isActive) { + enabledRuleArray[rule] = true; + + if (disabledRulesArray[rule] !== undefined) { + delete disabledRulesArray[rule]; + } + } else { + disabledRulesArray[rule] = true; + + if (enabledRuleArray[rule] !== undefined) { + delete enabledRuleArray[rule]; + } + } + }; + + /** + * Add a rule to the rule array + * and replace old rule with new rule. + * + * @param {String} rule RegExp as string + * @param {boolean} isActive Is this rule active? + */ + this.addRule = function (rule, isActive = true) { + rule = "([\\/\\?#]|(&|&))+(" + rule + "=[^&]*)"; + + this.applyRule(enabled_rules, disabled_rules, rule, isActive); + }; + + /** + * Return all active rules as an array. + * + * @return Array RegExp strings + */ + this.getRules = function () { + if (!storage.referralMarketing) { + return Object.keys(Object.assign(enabled_rules, enabled_referralMarketing)); + } + + return Object.keys(enabled_rules); + }; + + /** + * Add a raw rule to the raw rule array + * and replace old raw rule with new raw rule. + * + * @param {String} rule RegExp as string + * @param {boolean} isActive Is this rule active? + */ + this.addRawRule = function (rule, isActive = true) { + this.applyRule(enabled_rawRules, disabled_rawRules, rule, isActive); + }; + + /** + * Return all active raw rules as an array. + * + * @return Array RegExp strings + */ + this.getRawRules = function () { + return Object.keys(enabled_rawRules); + }; + + /** + * Add a referral marketing rule to the referral marketing array + * and replace old referral marketing rule with new referral marketing rule. + * + * @param {String} rule RegExp as string + * @param {boolean} isActive Is this rule active? + */ + this.addReferralMarketing = function (rule, isActive = true) { + rule = "([\\/\\?#]|(&|&))+(" + rule + "=[^\\/\\?&]*)"; + + this.applyRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive); + }; + + /** + * Add a exception to the exceptions array + * and replace old with new exception. + * + * @param {String} exception RegExp as string + * @param {Boolean} isActive Is this exception acitve? + */ + this.addException = function (exception, isActive = true) { + if (isActive) { + enabled_exceptions[exception] = true; + + if (disabled_exceptions[exception] !== undefined) { + delete disabled_exceptions[exception]; + } + } else { + disabled_exceptions[exception] = true; + + if (enabled_exceptions[exception] !== undefined) { + delete enabled_exceptions[exception]; + } + } + }; + + /** + * Private helper method to check if the url + * an exception. + * + * @param {String} url RegExp as string + * @return {boolean} if matching? true: false + */ + this.matchException = function (url) { + let result = false; + + //Add the site blocked alert to every exception + if (url === siteBlockedAlert) return true; + + for (const exception in enabled_exceptions) { + if (result) break; + + let exception_regex = new RegExp(exception, "i"); + result = exception_regex.test(url); + } + + return result; + }; + + /** + * Add a redirection to the redirections array + * and replace old with new redirection. + * + * @param {String} redirection RegExp as string + * @param {Boolean} isActive Is this redirection active? + */ + this.addRedirection = function (redirection, isActive = true) { + if (isActive) { + enabled_redirections[redirection] = true; + + if (disabled_redirections[redirection] !== undefined) { + delete disabled_redirections[redirection]; + } + } else { + disabled_redirections[redirection] = true; + + if (enabled_redirections[redirection] !== undefined) { + delete enabled_redirections[redirection]; + } + } + }; + + /** + * Return all redirection. + * + * @return url + */ + this.getRedirection = function (url) { + let re = null; + + for (const redirection in enabled_redirections) { + let result = (url.match(new RegExp(redirection, "i"))); + + if (result && result.length > 0 && redirection) { + re = (new RegExp(redirection, "i")).exec(url)[1]; + + break; + } + } + + return re; + }; + } + + // ################################################################## + + /** + * Function which called from the webRequest to + * remove the tracking fields from the url. + * + * @param {webRequest} request webRequest-Object + * @return {Array} redirectUrl or none + */ + function clearUrl(request) { + const URLbeforeReplaceCount = countFields(request.url); + + //Add Fields form Request to global url counter + increaseGlobalURLCounter(URLbeforeReplaceCount); + + if (storage.globalStatus) { + let result = { + "changes": false, + "url": "", + "redirect": false, + "cancel": false + }; + + if (storage.pingBlocking && storage.pingRequestTypes.includes(request.type)) { + pushToLog(request.url, request.url, translate('log_ping_blocked')); + increaseBadged(false, request); + increaseGlobalURLCounter(1); + return {cancel: true}; + } + + /* + * Call for every provider the removeFieldsFormURL method. + */ + for (let i = 0; i < providers.length; i++) { + if (providers[i].matchURL(request.url)) { + result = removeFieldsFormURL(providers[i], request.url, false, request); + } + + /* + * Expand urls and bypass tracking. + * Cancel the active request. + */ + if (result.redirect) { + if (providers[i].shouldForceRedirect() && + request.type === 'main_frame') { + browser.tabs.update(request.tabId, {url: result.url}).catch(handleError); + return {cancel: true}; + } + + return { + redirectUrl: result.url + }; + } + + /* + * Cancel the Request and redirect to the site blocked alert page, + * to inform the user about the full url blocking. + */ + if (result.cancel) { + if (request.type === 'main_frame') { + const blockingPage = browser.extension.getURL("html/siteBlockedAlert.html?source=" + encodeURIComponent(request.url)); + browser.tabs.update(request.tabId, {url: blockingPage}).catch(handleError); + + return {cancel: true}; + } else { + return { + redirectUrl: siteBlockedAlert + }; + } + } + + /* + * Ensure that the function go not into + * a loop. + */ + if (result.changes) { + return { + redirectUrl: result.url + }; + } + } + } + + // Default case + return {}; + } + + /** + * Call loadOldDataFromStore, getHash, counter, status and log functions + */ + + loadOldDataFromStore(); + getHash(); + setBadgedStatus(); + + /** + * Check the request. + */ + function promise(requestDetails) { + if (isDataURL(requestDetails)) { + return {}; + } else { + return clearUrl(requestDetails); + } + } + + /** + * To prevent long loading on data urls + * we will check here for data urls. + * + * @type {requestDetails} + * @return {boolean} + */ + function isDataURL(requestDetails) { + const s = requestDetails.url; + + return s.substring(0, 4) === "data"; + } + + /** + * Call by each Request and checking the url. + * + * @type {Array} + */ + browser.webRequest.onBeforeRequest.addListener( + promise, + {urls: [""], types: getData("types").concat(getData("pingRequestTypes"))}, + ["blocking"] + ); +} diff --git a/data/data.min.json b/data/data.min.json index 7fc92fb..50b04df 100644 --- a/data/data.min.json +++ b/data/data.min.json @@ -135,7 +135,8 @@ "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-.]*\\.)?(google)(\\.[a-zA-Z]{2,})\\/search\\?.*", "completeProvider": false, "rules": [ - "client" + "client", + "sclient" ], "referralMarketing": [], "rawRules": [], @@ -403,7 +404,18 @@ "reddit": { "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-.]*\\.)?(reddit)(\\.[a-zA-Z]{2,}).*", "completeProvider": false, - "rules": [], + "rules": [ + "%24deep_link", + "\\$deep_link", + "correlation_id", + "ref_campaign", + "ref_source", + "%243p", + "\\$3p", + "%24original_url", + "\\$original_url", + "_branch_match_id" + ], "referralMarketing": [], "rawRules": [], "exceptions": [], @@ -1667,6 +1679,35 @@ "exceptions": [], "redirections": [], "forceRedirection": false + }, + "bestbuy.com": { + "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-.]*\\.)?(bestbuy)(\\.com).*", + "completeProvider": false, + "rules": [ + "irclickid", + "irgwc", + "loc", + "acampID", + "mpid", + "intl" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "digidip.net": { + "urlPattern": "(https:\\/\\/|http:\\/\\/)([a-zA-Z0-9-.]*\\.)?(digidip)(\\.net).*", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + ".*url=([^&]*)" + ], + "forceRedirection": false } } }