Version 1.9.2

#290

.gitlab-ci.yml

@@ -4,6 +4,7 @@ before_script:
   - export DEBIAN_FRONTEND= noninteractive
   - apt-get update -y
   - apt-get install -y zip unzip jq
+  - apt-get install nodejs
 
 stages:
   - build
@@ -13,9 +14,13 @@ hash rules:
     stage: build
     script:
         - sha256sum data/data.min.json | awk '{print $1}' > rules.min.hash
+        - node build_tools/minifyDataJSON.js "..\data\data.min.json" "..\data\data.minify.json"
+        - sha256sum data/data.minify.json | awk '{print $1}' > rules.minify.hash
     artifacts:
         paths:
             - rules.min.hash
+            - data.minify.json
+            - rules.minify.hash
 
 build firefox:
   stage: build

CHANGELOG.md

@@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.9.2] - 2019-11-09
+
+### Compatibility note
+- Require Firefox >= 55
+- Require Chrome >= 22
+
+### Fixed
+- Fixed [#290](https://gitlab.com/KevinRoebert/ClearUrls/issues/290)
+
+## Changed
+- Updated some strings of Italian translation by [@gioxx](https://gitlab.com/gioxx)
+
+### Added
+- Added a minimal version of the data.min.json file where all line breaks and spaces, as well as default values and empty lists are removed.
+
 ## [1.9.1] - 2019-10-24
 
 ### Compatibility note

build_tools/minifyDataJSON.js

@@ -0,0 +1,87 @@
+/*
+* ClearURLs
+* Copyright (c) 2017-2019 Kevin Röbert
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*jshint esversion: 6 */
+/*
+* This script is responsible for minification of the data.min.json file and deletes also empty entries.
+*/
+let fs = require('fs');
+const inFileLocation = process.argv.slice(2)[0];
+const outFileLocation = process.argv.slice(2)[1];
+
+if(inFileLocation === undefined || outFileLocation === undefined) {
+    throw "in- and output must be set!";
+}
+
+const fileContent = fs.readFileSync(inFileLocation).toString();
+
+/**
+ * Builds a minify version of the data.min.json file.
+ */
+function build() {
+    const data = JSON.parse(fileContent);
+    let minifiedData = {"providers":{}};
+
+    for(let provider in data.providers) {
+        minifiedData.providers[provider] = {};
+        let self = minifiedData.providers[provider];
+
+        if(data.providers[provider].completeProvider === true) {
+            self.completeProvider = true;
+        }
+
+        if(data.providers[provider].forceRedirection === true) {
+            self.forceRedirection = true;
+        }
+
+        if(data.providers[provider].urlPattern !== "") {
+            self.urlPattern = data.providers[provider].urlPattern;
+        }
+
+        if(data.providers[provider].rules.length !== 0) {
+            self.rules = data.providers[provider].rules;
+        }
+
+        if(data.providers[provider].rawRules.length !== 0) {
+            self.rawRules = data.providers[provider].rawRules;
+        }
+
+        if(data.providers[provider].referralMarketing.length !== 0) {
+            self.referralMarketing = data.providers[provider].referralMarketing;
+        }
+
+        if(data.providers[provider].exceptions.length !== 0) {
+            self.exceptions = data.providers[provider].exceptions;
+        }
+
+        if(data.providers[provider].redirections.length !== 0) {
+            self.redirections = data.providers[provider].redirections;
+        }
+    }
+
+    fs.writeFile(outFileLocation, JSON.stringify(minifiedData), function(err) {
+
+        if(err) {
+            return console.log(err);
+        }
+
+        console.log("The file was saved!");
+    });
+}
+
+build();

core_js/storage.js

@@ -25,10 +25,9 @@ var hasPendingSaves = false;
 var pendingSaves = new Set();
 
 /**
-* Writes the storage variable to the disk.
+ * Writes the storage variable to the disk.
-*/
+ */
-function saveOnExit()
+function saveOnExit() {
-{
     saveOnDisk(Object.keys(storage));
 }
 
@@ -65,14 +64,13 @@ function storageDataAsString(key) {
 }
 
 /**
-* Save multiple keys on the disk.
+ * Save multiple keys on the disk.
-* @param  {String[]} keys
+ * @param  {String[]} keys
-*/
+ */
-function saveOnDisk(keys)
+function saveOnDisk(keys) {
-{
     let json = {};
 
-    keys.forEach(function(key) {
+    keys.forEach(function (key) {
         json[key] = storageDataAsString(key);
     });
 
@@ -81,17 +79,16 @@ function saveOnDisk(keys)
 }
 
 /**
-* Schedule to save a key to disk in 30 seconds.
+ * Schedule to save a key to disk in 30 seconds.
-* @param  {String} key
+ * @param  {String} key
-*/
+ */
-function deferSaveOnDisk(key)
+function deferSaveOnDisk(key) {
-{
     if (hasPendingSaves) {
         pendingSaves.add(key);
         return;
     }
 
-    setTimeout(function() {
+    setTimeout(function () {
         saveOnDisk(Array.from(pendingSaves));
         pendingSaves.clear();
         hasPendingSaves = false;
@@ -100,10 +97,9 @@ function deferSaveOnDisk(key)
 }
 
 /**
-* Start sequence for ClearURLs.
+ * Start sequence for ClearURLs.
-*/
+ */
-function genesis()
+function genesis() {
-{
     browser.storage.local.get(null).then((items) => {
         initStorage(items);
 
@@ -119,35 +115,32 @@ function genesis()
 }
 
 /**
-* Return the value under the key.
+ * Return the value under the key.
-* @param  {String} key
+ * @param  {String} key
-* @return {Object}
+ * @return {Object}
-*/
+ */
-function getData(key)
+function getData(key) {
-{
     return storage[key];
 }
 
 /**
-* Return the entire storage object.
+ * Return the entire storage object.
-* @return {Object}
+ * @return {Object}
-*/
+ */
-function getEntireData()
+function getEntireData() {
-{
     return storage;
 }
 
 /**
-* Save the value under the key on the RAM.
+ * Save the value under the key on the RAM.
-*
+ *
-* Note: To store the data on the hard disk, one of
+ * Note: To store the data on the hard disk, one of
-*  deferSaveOnDisk(), saveOnDisk(), or saveOnExit()
+ *  deferSaveOnDisk(), saveOnDisk(), or saveOnExit()
-*  must be called.
+ *  must be called.
-* @param {String} key
+ * @param {String} key
-* @param {Object} value
+ * @param {Object} value
-*/
+ */
-function setData(key, value)
+function setData(key, value) {
-{
     switch (key) {
         case "ClearURLsData":
         case "log":
@@ -169,23 +162,21 @@ function setData(key, value)
 }
 
 /**
-* Write error on console.
+ * Write error on console.
-*/
+ */
-function error(e)
+function error(e) {
-{
     console.log(translate('core_error'));
     console.error(e);
 }
 
 /**
-* Set default values, if the storage is empty.
+ * Set default values, if the storage is empty.
-* @param  {Object} items
+ * @param  {Object} items
-*/
+ */
-function initStorage(items)
+function initStorage(items) {
-{
     initSettings();
 
-    if(!isEmpty(items)) {
+    if (!isEmpty(items)) {
         Object.entries(items).forEach(([key, value]) => {
             setData(key, value);
         });
@@ -193,10 +184,9 @@ function initStorage(items)
 }
 
 /**
-* Set default values for the settings.
+ * Set default values for the settings.
-*/
+ */
-function initSettings()
+function initSettings() {
-{
     storage.ClearURLsData = [];
     storage.dataHash = "";
     storage.badgedStatus = true;
@@ -208,15 +198,15 @@ function initSettings()
     storage.log = {"log": []};
     storage.statisticsStatus = true;
     storage.badged_color = "ffa500";
-    storage.hashURL = "https://gitlab.com/KevinRoebert/ClearUrls/-/jobs/artifacts/master/raw/rules.min.hash?job=hash%20rules";
+    storage.hashURL = "https://gitlab.com/KevinRoebert/ClearUrls/-/jobs/artifacts/master/raw/rules.minify.hash?job=hash%20rules";
-    storage.ruleURL = "https://gitlab.com/KevinRoebert/ClearUrls/raw/master/data/data.min.json";
+    storage.ruleURL = "https://gitlab.com/KevinRoebert/ClearUrls/raw/master/data/data.minify.json";
     storage.contextMenuEnabled = true;
     storage.historyListenerEnabled = true;
     storage.localHostsSkipping = true;
     storage.referralMarketing = false;
     storage.logLimit = -1;
 
-    if(getBrowser() === "Firefox") {
+    if (getBrowser() === "Firefox") {
         storage.types = ["font", "image", "imageset", "main_frame", "media", "object", "object_subrequest", "other", "script", "stylesheet", "sub_frame", "websocket", "xbl", "xml_dtd", "xmlhttprequest", "xslt"];
     } else if (getBrowser() === "Chrome") {
         storage.types = ["main_frame", "sub_frame", "stylesheet", "script", "image", "font", "object", "xmlhttprequest", "ping", "csp_report", "media", "websocket", "other"];
@@ -224,53 +214,57 @@ function initSettings()
 }
 
 /**
-* Replace the old URLs with the
+ * Replace the old URLs with the
-* new GitLab URLs.
+ * new GitLab URLs.
-*/
+ */
-function replaceOldURLs(url)
+function replaceOldURLs(url) {
-{
     switch (url) {
         case "https://raw.githubusercontent.com/KevinRoebert/ClearUrls/master/data/rules.hash?flush_cache=true":
-        return "https://gitlab.com/KevinRoebert/ClearUrls/-/jobs/artifacts/master/raw/rules.min.hash?job=hash%20rules";
+            return "https://gitlab.com/KevinRoebert/ClearUrls/-/jobs/artifacts/master/raw/rules.min.hash?job=hash%20rules";
         case "https://raw.githubusercontent.com/KevinRoebert/ClearUrls/master/data/data.json?flush_cache=true":
-        return "https://gitlab.com/KevinRoebert/ClearUrls/raw/master/data/data.min.json";
+            return "https://gitlab.com/KevinRoebert/ClearUrls/raw/master/data/data.min.json";
         case "https://gitlab.com/KevinRoebert/ClearUrls/raw/master/data/rules.hash":
-        return "https://gitlab.com/KevinRoebert/ClearUrls/-/jobs/artifacts/master/raw/rules.min.hash?job=hash%20rules";
+            return "https://gitlab.com/KevinRoebert/ClearUrls/-/jobs/artifacts/master/raw/rules.min.hash?job=hash%20rules";
         case "https://gitlab.com/KevinRoebert/ClearUrls/raw/master/data/data.json":
-        return "https://gitlab.com/KevinRoebert/ClearUrls/raw/master/data/data.min.json";
+            return "https://gitlab.com/KevinRoebert/ClearUrls/raw/master/data/data.min.json";
+        case "https://gitlab.com/KevinRoebert/ClearUrls/-/jobs/artifacts/master/raw/rules.min.hash?job=hash%20rules":
+            return "https://gitlab.com/KevinRoebert/ClearUrls/-/jobs/artifacts/master/raw/rules.minify.hash?job=hash%20rules";
+        case "https://gitlab.com/KevinRoebert/ClearUrls/raw/master/data/data.min.json":
+            return "https://gitlab.com/KevinRoebert/ClearUrls/-/jobs/artifacts/master/raw/data.minify.json?job=hash%20rules";
         default:
-        return url;
+            return url;
     }
 }
 
 /**
-* Load local saved data, if the browser is offline or
+ * Load local saved data, if the browser is offline or
-* some other network trouble.
+ * some other network trouble.
-*/
+ */
-function loadOldDataFromStore()
+function loadOldDataFromStore() {
-{
     localDataHash = storage.dataHash;
 }
 
 /**
-* Save the hash status to the local storage (RAM).
+ * Save the hash status to the local storage (RAM).
-* The status can have the following values:
+ * The status can have the following values:
-*  1 "up to date"
+ *  1 "up to date"
-*  2 "updated"
+ *  2 "updated"
-*  3 "update available"
+ *  3 "update available"
-*  @param status_code the number for the status
+ *  @param status_code the number for the status
-*/
+ */
-function storeHashStatus(status_code)
+function storeHashStatus(status_code) {
-{
+    switch (status_code) {
-    switch(status_code)
+        case 1:
-    {
+            status_code = "hash_status_code_1";
-        case 1: status_code = "hash_status_code_1";
+            break;
-        break;
+        case 2:
-        case 2: status_code = "hash_status_code_2";
+            status_code = "hash_status_code_2";
-        break;
+            break;
-        case 3: status_code = "hash_status_code_3";
+        case 3:
-        break;
+            status_code = "hash_status_code_3";
-        default: status_code = "hash_status_code_4";
+            break;
+        default:
+            status_code = "hash_status_code_4";
     }
 
     storage.hashStatus = status_code;

core_js/tools.js

@@ -24,53 +24,49 @@
 /*
 * To support Waterfox.
 */
-Array.prototype.rmEmpty = function() {
+Array.prototype.rmEmpty = function () {
     return this.filter(v => v);
 };
 
 /*
 * To support Waterfox.
 */
-Array.prototype.flatten = function() {
+Array.prototype.flatten = function () {
     return this.reduce((a, b) => a.concat(b), []);
 };
 
 /**
-* Check if an object is empty.
+ * Check if an object is empty.
-* @param  {Object}  obj
+ * @param  {Object}  obj
-* @return {Boolean}
+ * @return {Boolean}
-*/
+ */
-function isEmpty(obj)
+function isEmpty(obj) {
-{
     return (Object.getOwnPropertyNames(obj).length === 0);
 }
 
 /**
-* Translate a string with the i18n API.
+ * Translate a string with the i18n API.
-*
+ *
-* @param {string} string Name of the attribute used for localization
+ * @param {string} string Name of the attribute used for localization
-*/
+ */
-function translate(string)
+function translate(string) {
-{
     return browser.i18n.getMessage(string);
 }
 
 /**
-* Reloads the extension.
+ * Reloads the extension.
-*/
+ */
-function reload()
+function reload() {
-{
     browser.runtime.reload();
 }
 
 /**
-* Check if it is an android device.
+ * Check if it is an android device.
-* @return bool
+ * @return bool
-*/
+ */
-async function checkOSAndroid()
+async function checkOSAndroid() {
-{
+    if (os === undefined || os === null || os === "") {
-    if(os === undefined || os === null || os === "") {
+        await chrome.runtime.getPlatformInfo(function (info) {
-        await chrome.runtime.getPlatformInfo(function(info) {
             os = info.os;
         });
     }
@@ -79,10 +75,10 @@ async function checkOSAndroid()
 }
 
 /**
-* Extract the host without port from an url.
+ * Extract the host without port from an url.
-* @param  {String} url URL as String
+ * @param  {String} url URL as String
-* @return {String}     host as string
+ * @return {String}     host as string
-*/
+ */
 function extractHost(url) {
     let parsed_url = new URL(url);
 
@@ -90,36 +86,34 @@ function extractHost(url) {
 }
 
 /**
-* Returns true if the url has a local host.
+ * Returns true if the url has a local host.
-* @param  {String} url URL as String
+ * @param  {String} url URL as String
-* @return {boolean}
+ * @return {boolean}
-*/
+ */
 function checkLocalURL(url) {
     let host = extractHost(url);
 
     return ipRangeCheck(host, ["10.0.0.0/8", "172.16.0.0/12",
-    "192.168.0.0/16", "100.64.0.0/10",
+            "192.168.0.0/16", "100.64.0.0/10",
-    "169.254.0.0/16", "127.0.0.1"])||
+            "169.254.0.0/16", "127.0.0.1"]) ||
-    host === 'localhost';
+        host === 'localhost';
 }
 
 /**
-* Return the number of parameters query strings.
+ * Return the number of parameters query strings.
-* @param  {String}     url URL as String
+ * @param  {String}     url URL as String
-* @return {int}        Number of Parameters
+ * @return {int}        Number of Parameters
-*/
+ */
-function countFields(url)
+function countFields(url) {
-{
     return extractFileds(url).length;
 }
 
 /**
-* Returns true if fields exists.
+ * Returns true if fields exists.
-* @param  {String}     url URL as String
+ * @param  {String}     url URL as String
-* @return {boolean}
+ * @return {boolean}
-*/
+ */
-function existsFields(url)
+function existsFields(url) {
-{
     let matches = (url.match(/\?.+/i) || []);
     let count = matches.length;
 
@@ -127,15 +121,14 @@ function existsFields(url)
 }
 
 /**
-* Extract the fields from an url.
+ * Extract the fields from an url.
-* @param  {String} url URL as String
+ * @param  {String} url URL as String
-* @return {Array}     Fields as array
+ * @return {Array}     Fields as array
-*/
+ */
-function extractFileds(url)
+function extractFileds(url) {
-{
+    if (existsFields(url)) {
-    if(existsFields(url)) {
         let fields = url.replace(new RegExp(".*?\\?", "i"), "");
-        if(existsFragments(url)) {
+        if (existsFragments(url)) {
             fields = fields.replace(new RegExp("#.*", "i"), "");
         }
 
@@ -146,23 +139,21 @@ function extractFileds(url)
 }
 
 /**
-* Return the number of fragments query strings.
+ * Return the number of fragments query strings.
-* @param  {String}     url URL as String
+ * @param  {String}     url URL as String
-* @return {int}        Number of fragments
+ * @return {int}        Number of fragments
-*/
+ */
-function countFragments(url)
+function countFragments(url) {
-{
     return extractFragments(url).length;
 }
 
 /**
-* Extract the fragments from an url.
+ * Extract the fragments from an url.
-* @param  {String} url URL as String
+ * @param  {String} url URL as String
-* @return {Array}     fragments as array
+ * @return {Array}     fragments as array
-*/
+ */
-function extractFragments(url)
+function extractFragments(url) {
-{
+    if (existsFragments(url)) {
-    if(existsFragments(url)) {
         let fragments = url.replace(new RegExp(".*?#", "i"), "");
         return (fragments.match(/[^&]+=?[^&]*/gi) || []);
     }
@@ -171,12 +162,11 @@ function extractFragments(url)
 }
 
 /**
-* Returns true if fragments exists.
+ * Returns true if fragments exists.
-* @param  {String}     url URL as String
+ * @param  {String}     url URL as String
-* @return {boolean}
+ * @return {boolean}
-*/
+ */
-function existsFragments(url)
+function existsFragments(url) {
-{
     let matches = (url.match(/\#.+/i) || []);
     let count = matches.length;
 
@@ -184,73 +174,69 @@ function existsFragments(url)
 }
 
 /**
-* Load local saved data, if the browser is offline or
+ * Load local saved data, if the browser is offline or
-* some other network trouble.
+ * some other network trouble.
-*/
+ */
-function loadOldDataFromStore()
+function loadOldDataFromStore() {
-{
     localDataHash = storage.dataHash;
 }
 
 /**
-* Save the hash status to the local storage (RAM).
+ * Save the hash status to the local storage (RAM).
-* The status can have the following values:
+ * The status can have the following values:
-*  1 "up to date"
+ *  1 "up to date"
-*  2 "updated"
+ *  2 "updated"
-*  3 "update available"
+ *  3 "update available"
-*  @param status_code the number for the status
+ *  @param status_code the number for the status
-*/
+ */
-function storeHashStatus(status_code)
+function storeHashStatus(status_code) {
-{
+    switch (status_code) {
-    switch(status_code)
+        case 1:
-    {
+            status_code = "hash_status_code_1";
-        case 1: status_code = "hash_status_code_1";
+            break;
-        break;
+        case 2:
-        case 2: status_code = "hash_status_code_2";
+            status_code = "hash_status_code_2";
-        break;
+            break;
-        case 3: status_code = "hash_status_code_3";
+        case 3:
-        break;
+            status_code = "hash_status_code_3";
-        default: status_code = "hash_status_code_4";
+            break;
+        default:
+            status_code = "hash_status_code_4";
     }
 
     storage.hashStatus = status_code;
 }
 
 /**
-* Increase by {number} the GlobalURLCounter
+ * Increase by {number} the GlobalURLCounter
-* @param  {int} number
+ * @param  {int} number
-*/
+ */
-function increaseGlobalURLCounter(number)
+function increaseGlobalURLCounter(number) {
-{
+    if (storage.statisticsStatus) {
-    if(storage.statisticsStatus)
-    {
         storage.globalurlcounter += number;
         deferSaveOnDisk('globalurlcounter');
     }
 }
 
 /**
-* Increase by one the URLCounter
+ * Increase by one the URLCounter
-*/
+ */
-function increaseURLCounter()
+function increaseURLCounter() {
-{
+    if (storage.statisticsStatus) {
-    if(storage.statisticsStatus)
-    {
         storage.globalCounter++;
         deferSaveOnDisk('globalCounter');
     }
 }
 
 /**
-* Change the icon.
+ * Change the icon.
-*/
+ */
-function changeIcon()
+function changeIcon() {
-{
     checkOSAndroid().then((res) => {
-        if(!res) {
+        if (!res) {
-            if(storage.globalStatus){
+            if (storage.globalStatus) {
                 browser.browserAction.setIcon({path: "img/clearurls_128x128.png"});
-            } else{
+            } else {
                 browser.browserAction.setIcon({path: "img/clearurls_gray_128x128.png"});
             }
         }
@@ -258,35 +244,33 @@ function changeIcon()
 }
 
 /**
-* Get the badged status from the browser storage and put the value
+ * Get the badged status from the browser storage and put the value
-* into a local variable.
+ * into a local variable.
-*
+ *
-*/
+ */
-function setBadgedStatus()
+function setBadgedStatus() {
-{
     checkOSAndroid().then((res) => {
-        if(!res && storage.badgedStatus) {
+        if (!res && storage.badgedStatus) {
             browser.browserAction.setBadgeBackgroundColor({
-                'color': '#'+storage.badged_color
+                'color': '#' + storage.badged_color
             });
         }
     });
 }
 
 /**
-* Returns the current URL.
+ * Returns the current URL.
-* @return {String} [description]
+ * @return {String} [description]
-*/
+ */
-function getCurrentURL()
+function getCurrentURL() {
-{
     return currentURL;
 }
 
 /**
-* Check for browser.
+ * Check for browser.
-*/
+ */
 function getBrowser() {
-    if(typeof InstallTrigger !== 'undefined') {
+    if (typeof InstallTrigger !== 'undefined') {
         return "Firefox";
     } else {
         return "Chrome";
@@ -299,9 +283,19 @@ function getBrowser() {
  */
 function decodeURL(url) {
     const rtn = decodeURIComponent(url);
-    if(rtn.indexOf("http://") === -1 && rtn.indexOf("https://") === -1) {
+    if (rtn.indexOf("http://") === -1 && rtn.indexOf("https://") === -1) {
         return decodeURL(rtn);
     }
 
     return rtn;
 }
+
+/*
+* Gets the value of at `key` an object. If the resolved value is `undefined`, the `defaultValue` is returned in its place.
+*
+* @param {string} key the key of the object
+* @param {object} defaultValue the default value
+*/
+Object.prototype.getOrDefault = function (key, defaultValue) {
+    return this[key] === undefined ? defaultValue : this[key];
+};

data/data.min.json

@@ -36,7 +36,7 @@
                 "rnid"
             ],
             "referralMarketing": [
-                "([\\/|\\?|#]|(&|&))+(tag=[^\\/|\\?|&]*)"
+                "tag"
             ],
             "exceptions": [
                 ".*(amazon\\.).*(\\/gp).*\\/redirector.html\\/.*",

manifest.json

@@ -1,7 +1,7 @@
 {
     "manifest_version": 2,
     "name": "ClearURLs",
-    "version": "1.9.1",
+    "version": "1.9.2",
     "author": "Kevin Röbert",
     "description": "Remove tracking elements from URLs.",
     "homepage_url": "https://gitlab.com/KevinRoebert/ClearUrls",