19 files changed, 246 insertions, 205 deletions

diff --git a/banapedia/Ban.py b/banapedia/Ban.py
deleted file mode 100644
index 4714274..0000000
--- a/banapedia/Ban.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from banapedia.wapi.WikipediaQuery import BlockQuery
-from datetime import datetime
-import pygeoip
-
-__author__ = 'pacien'
-
-
-GEOIP_FILE = "/usr/share/GeoIP/GeoIP.dat"
-geoip = pygeoip.GeoIP(GEOIP_FILE)
-
-ISO_TIMESTAMP = "%Y-%m-%dT%H:%M:%SZ"
-
-
-class Ban:
-    def __init__(self, ip, start, end):
-        self.ip = ip
-        self.start = start
-        self.end = end
-        self.country_code = None
-
-    def get_duration(self):
-        return (self.end - self.start).days
-
-    def get_country_code(self):
-        if self.country_code is not None:
-            return self.country_code
-
-        country_code = ""
-
-        try:
-            country_code = geoip.country_code_by_addr(self.ip).lower()
-        except pygeoip.GeoIPError:
-            print("[ERROR]", "Could not determine country for ip", self.ip)
-
-        self.country_code = country_code
-        return country_code
diff --git a/banapedia/__init__.py b/banapedia/__init__.py
deleted file mode 100644
index 8b13789..0000000
--- a/banapedia/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/banapedia/api/__init__.py b/banapedia/api/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/banapedia/api/__init__.py
+++ /dev/null
diff --git a/banapedia/wapi/WikipediaQuery.py b/banapedia/wapi/WikipediaQuery.py
deleted file mode 100644
index d3d2f94..0000000
--- a/banapedia/wapi/WikipediaQuery.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from ..api.Query import JSONQuery
-
-__author__ = 'pacien'
-
-WIKIPEDIA_QUERY_BASE_URL = "https://en.wikipedia.org/w/api.php"
-LIST_SEPARATOR = "|"
-DEFAULT_BKPROP = ["id", "user", "userid", "by", "byid", "timestamp", "expiry", "reason", "range", "flags"]
-DEFAULT_BKSHOW = ["account", "temp", "ip", "range"]
-
-
-class WikipediaQuery(JSONQuery):
-    def __init__(self, params={}):
-        params.update({
-            "action": "query",
-            "format": "json",
-        })
-        JSONQuery.__init__(self, base_url=WIKIPEDIA_QUERY_BASE_URL, params=params)
-
-
-class ListQuery(WikipediaQuery):
-    def __init__(self, list_name, params={}):
-        params.update({
-            "list": list_name,
-        })
-        WikipediaQuery.__init__(self, params)
-
-
-class BlockQuery(ListQuery):
-    def __init__(self, bkprop=DEFAULT_BKPROP, bkshow=DEFAULT_BKSHOW, bkdir="newer", limit=500, continue_token=None):
-        params = {
-            "bkprop": LIST_SEPARATOR.join(bkprop),
-            "bkshow": LIST_SEPARATOR.join(bkshow),
-            "bkdir": bkdir,
-            "bklimit": limit,
-        }
-
-        if continue_token is not None:
-            params.update({"bkcontinue": continue_token})
-
-        ListQuery.__init__(self, "blocks", params=params)
-
-
diff --git a/banapedia/wapi/__init__.py b/banapedia/wapi/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/banapedia/wapi/__init__.py
+++ /dev/null
diff --git a/bandict/__init__.py b/bandict/__init__.py
deleted file mode 100644
index 389ae0e..0000000
--- a/bandict/__init__.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import urllib.request
-import json
-import numpy as np
-from banapedia.wapi.WikipediaQuery import BlockQuery
-from banapedia.Ban import *
-
-
-class BanList():
-
-    def __init__(self, data_file, samples=30000, samples_by_query=500,
-                 from_internet=False):
-        if from_internet:
-            proxy = urllib.request.ProxyHandler(urllib.request.getproxies())
-            opener = urllib.request.build_opener(proxy)
-            urllib.request.install_opener(opener)
-            self.dict_list = self.fetch_multipart(samples, samples_by_query)
-        else:
-            with open(data_file, "r") as ban_dict_file:
-                self.dict_list = json.load(ban_dict_file)
-        self.ban_list = []
-        for ban_dict in self.dict_list:
-            self.ban_list.append(Ban(
-                ban_dict["user"],
-                datetime.strptime(ban_dict["timestamp"], ISO_TIMESTAMP),
-                datetime.strptime(ban_dict["expiry"], ISO_TIMESTAMP),
-            ))
-
-    def fetch_multipart(self, n, query_limit):
-        ban_dict_list = []
-        n_fetched = 0
-        continue_token = None
-
-        print("[INFO]", "Fetching %d bans" % n)
-        while n_fetched < n:
-            to_fetch = min(query_limit, n - n_fetched)
-            query = BlockQuery(
-                bkprop=["user", "timestamp", "expiry"],
-                bkshow=["temp", "ip"],
-                limit=to_fetch,
-                continue_token=continue_token,
-            )
-            results = query.fetch_result()
-            ban_dict_list.extend(results["query"]["blocks"])
-            continue_token = results["query-continue"]["blocks"]["bkcontinue"]
-            n_fetched += to_fetch
-            print("[INFO]", "Fetched %d over %d bans" % (n_fetched, n))
-
-        print("[INFO]", "Bans fetching complete")
-        return ban_dict_list
-
-    def write_to_file(self, outfile):
-        with open(outfile, "w") as ban_dict_file:
-            json.dump(self.dict_list, ban_dict_file, indent="\t")
-
-    def get_durations(self):
-        return [ban.get_duration() for ban in self.ban_list]
-
-    def get_countries(self):
-        return [ban.get_country_code() for ban in self.ban_list]
-
-    def __iter__(self):
-        return self.dict_list.__iter__()
-
-    def items(self):
-        return self.dict_list.items()
-
-    def by_country(self):
-        ban_duration_by_country = {}
-
-        for ban in self.ban_list:
-            country_code = ban.get_country_code()
-
-            if country_code not in ban_duration_by_country.keys():
-                ban_duration_by_country[country_code] = []
-
-            ban_duration_by_country[country_code].append(ban)
-
-        return ban_duration_by_country
-
-    def average_ban_by_country(self):
-        average_ban_duration_ctry = {}
-
-        for country, bans in self.by_country().items():
-            average = np.mean([ban.get_duration() for ban in bans])
-            average_ban_duration_ctry[country] = average
-
-        # In months
-        average_ban_duration_ctry = {country: duration/30
-                                     for country, duration in
-                                     average_ban_duration_ctry.items()}
-        return average_ban_duration_ctry
diff --git a/main.py b/main.py
index 0d12a1e..da7c6ae 100644
--- a/main.py
+++ b/main.py
@@ -1,62 +1,101 @@
-from banapedia.Ban import *
-import bandict
 from collections import Counter
+import webbrowser
+
 import pygal
 import numpy as np
+import pygeoip
+
+from wikibania.ban.BanDB import BanDB
+from wikibania.ban.BanDBWrapper import BanDBWrapper
+import sysproxy
+
+
+# PARAMS
+
+GEOIP_DB = "/usr/share/GeoIP/GeoIP.dat"
 
 BAN_MAP_FILE = "output/ban-map.svg"
 BAN_DURATION_MAP_FILE = "output/ban-duration-map.svg"
 HIST_FILE = "output/histogram.svg"
+STATS_FILE = "output/stats.txt"
+
+BAN_DB_FILE = "resources/ban_list.json"
+
+FETCH_SAMPLES = 2000
+FETCH_DB = False
+DUMP_DB = False
+LOAD_DB = False
+
+OPEN_FILES = False
+
+
+# SETUP
 
-SAMPLES = 1000
+sysproxy.configure_system_proxy()
+geoip_looker = pygeoip.GeoIP(GEOIP_DB)
 
-BAN_FILE = "resources/ban_list.json"
+ban_db = BanDB(geoip_lo