diff --git a/.gitignore b/.gitignore
index f396aa8..ccc1003 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,4 +12,10 @@ workspace.code-workspace
password_arango.txt
*.txt
~requirements.txt
-*.gexf
\ No newline at end of file
+*.gexf
+facebook/mrkoll.py
+*.pyc
+facebook/tortest.py
+facebook/phone.py
+facebook/accs.py
+facebook/gephi.py
diff --git a/Dockerfile b/Dockerfile
index 4bdb66a..8bc5183 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,11 +3,13 @@ FROM python:3.8
WORKDIR /
+COPY data/ .
+
COPY requirements.txt .
RUN pip install -r requirements.txt
-ADD . .
+COPY facebook/accs.py facebook/classes.py facebook/config.py facebook/helpers.py facebook/__main__.py facebook/arangodb.py facebook/scrapers.py /facebook/
ENTRYPOINT [ "python", "facebook/__main__.py" ]
@@ -15,7 +17,5 @@ CMD ["",""]
# BUILD:
# docker buildx create --use
-#docker buildx build --platform linux/arm -t l3224/fb-reactions:pi --push .
+#docker buildx build --platform linux/arm,linux/arm64 -t l3224/fb-scraper:pi --push .
-# START
-# docker run -it --name fb1 -v vol1:/data l3224/fb-reactions:latest [-s -u user1,user2]
\ No newline at end of file
diff --git a/docker/free/Dockerfile b/docker/free/Dockerfile
new file mode 100644
index 0000000..c289f9d
--- /dev/null
+++ b/docker/free/Dockerfile
@@ -0,0 +1,19 @@
+
+FROM python:3.8
+
+WORKDIR /
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt
+
+ADD . .
+
+ENTRYPOINT [ "python", "facebook/__main__.py", "-p free" ]
+
+CMD ["",""]
+
+# BUILD:
+# docker buildx create --use
+#docker buildx build --file docker/free/Dockerfile --platform linux/arm -t l3224/fb-scraper:free --push .
+
diff --git a/docker/mrkoll/Dockerfile b/docker/mrkoll/Dockerfile
new file mode 100644
index 0000000..d61b23a
--- /dev/null
+++ b/docker/mrkoll/Dockerfile
@@ -0,0 +1,14 @@
+
+FROM python:3.8
+
+WORKDIR /
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt
+
+ADD . .
+
+ENTRYPOINT [ "python", "facebook/mrkoll.py" ]
+
+# docker buildx build --file docker/mrkoll/Dockerfile --platform linux/arm -t l3224/fb-scraper:mrkoll --push .
\ No newline at end of file
diff --git a/facebook/__main__.py b/facebook/__main__.py
index 3af1d13..13ab25d 100644
--- a/facebook/__main__.py
+++ b/facebook/__main__.py
@@ -1,221 +1,227 @@
import random
import traceback
-from getopt import GetoptError, getopt
-from sys import argv, exit
+from getopt import getopt
+from sys import argv
from time import sleep
-from subprocess import check_output
-from re import split
-from socket import gethostname
-
-from arangodb import db, write_report, backup, report_blocked, get_profile, remove_profile, checked_members, friends_of_user
-from classes import Profile, User
-from helpers import sleep_, write_error, _print
-from scrapers import profile_picture_reactions
-
-
-def finish():
- """ Avslutar: skriver rapport och gör profilerna oanvända """
- for profile in profiles:
- profile.unused()
- write_report(users, list(all_pictures.difference(all_pictures_start)))
- exit()
+from datetime import datetime
+from config import set_pwd
+from random import randint
if __name__ == "__main__":
print()
-
- if gethostname() not in ['macbook.local']: # Lägg till för studiodatorn
- # Hämta namn för containern där skriptet körs
- try:
- containers = check_output(['docker', 'container', 'ls']).decode()
- container = split('\W\W+', containers.split('\n')[1])[-1]
- except FileNotFoundError:
- pass
- else:
- container_name = 'macbook'
+ proxieservers = 'mullvad'
# Argument och alternativ
+
+ # Variabler som kan ändras
+ url_other_pictures = [] # Fylls eventuellt på
+ test = False
+ write = True
+ mode = 'all'
+ pwd = None
+ proxieservers = 'mullvad'
+
argv = argv[1:]
- try:
- opts, args = getopt(argv, "bm:u:o:", ['backup=',"mode=", "users=", "other="])
- for o, a in opts:
- # mode_nr används för hur ofta profile ska roteras
- if o in ["-m", "--mode"]:
- mode = a
- if mode == 'single':
- mode_nr = 1.7
- elif mode == 'few':
- mode_nr = 1.4
- elif mode == 'force':
- mode_nr = 1
- else:
- mode = 'all'
+
+ opts, args = getopt(argv, "bim:u:o:p:wl:", ["backup", "images", "mode=", "user=", "other=", "profiles=", "write", "password="])
+
+ for o, a in opts:
+ print(o)
+ if o in ['-l', "--password"]:
+ pwd = a.strip()
+
+ set_pwd(pwd)
+
+ # Importera andra moduler
+ from config import url_bas
+ from arangodb import (
+ blocked_profile,
+ new_profile,
+ backup,
+ get_user,
+ check_for_user,
+ friends_of_user,
+ )
+ from classes import Profile, User
+ from helpers import sleep_, write_error, _print, check_profile_status, update_cookie
+ from scrapers import profile_picture_reactions
+
+ for o, a in opts:
+
+ # Bestäm vilka profiler/proxies som ska användas
+ if o in ['-p', '--profiles']:
+ proxieservers = a.strip()
+ print(f'Proxieservers: {proxieservers}')
+
+ # Bestäm mode
+ if o in ["-m", "--mode"]:
+ mode = a.strip()
+ if mode == "single":
+ mode_nr = 1.7
+ elif mode == "few":
+ mode_nr = 1.4
+ elif mode == "solo":
+ mode_nr = 1.4
+ elif mode == "force":
mode_nr = 1
-
- for o, a in opts:
- if o in ["-u", "--user"]:
- try:
- users = [
- User(str(i).strip(), mode)
- for i in [(str(i).strip()) for i in a.split(",")]
- ]
- except StopIteration:
- raise Exception
- if o in ["-o", "--other"]:
- url_other_picture = a
- if o in ['-b', '--backup']:
- while True:
- backup(db)
- sleep(21600)
-
-
- if "users" not in globals():
- users = [
- User(str(i).strip(), mode)
- for i in input("Vem/vilka vill du kolla bilder för? ").split(",")
- ]
-
- except GetoptError:
- users = [
- User(str(i).strip(), mode)
- for i in input("Vem/vilka vill du kolla bilder för? ").split(",")
- ]
-
- mode = input("Söka efter alla, första/sida eller första? (all, few, single)? ").lower().strip()
- if mode == '':
- mode = 'all'
+
+ # Bestäm user
+ if o in ["-u", "--user"]:
+ if a == 'test': # För att testa profiler i profiles_test
+ test = True
+ container = str(a.strip())
+ if all([a.strip()[:4] == "leak", len(a) < 7]) or a == 'test':
+ sleep(randint(0, 40)/10) # För att docker service inte ska gå vidare exakt samtidigt
+ lookups = "leak_lookups"
+ userdoc = get_user(collection=lookups)
+ elif a.strip()[:7] == "lookups":
+ lookups = "lookups"
+ userdoc = get_user(collection=lookups)
+ if 'other' in userdoc:
+ url_other_pictures = userdoc['other']
+ else:
+ url_other_pictures = []
+ elif a == 'test':
+ lookups = "leak_lookups"
+ userdoc = get_user(collection=lookups)
+ else:
+ lookups = "lookups"
+ userdoc = {'_key': a}
- if "url_other_picture" in globals():
- users[0].url_other_picture = url_other_picture[url_other_picture.find('facebook.com') + 12:]
+ if o in ["-o", "--other"]:
+ url_other_pictures = a.split(",")
+ if o in ["-b", "--backup"]:
+ backup(db)
+ if o in ['-w', "--write"]:
+ write = False
+
+
+ if 'userdoc' not in globals():
+ lookups = "lookups"
+ userdoc = {'_key': str(input("Vem/vilka vill du kolla bilder för? ")).strip()}
- print("Kollar profilbilder för:")
- for user in users:
- print("-", user.username)
- print()
- if 'container' not in globals():
- usernames = [user.username for user in users]
- if len(usernames) == 1:
- container = usernames[0]
- else:
- container = '-'.join(usernames)
-
- # Skapa tre olika profiler att besöka Facebook med
- profiles = []
- for i in range(0, 3):
- doc = get_profile()
- profile = Profile(doc, container)
- profile.browser.open("https://api.ipify.org")
- print(f"Profil {profile.name} använder IP-adress {profile.viewing().text}."
- )
- if profile.logged_in == False:
- profile.accept_cookies()
- sleep_(2)
- profile.login()
- profiles.append(profile)
- print()
- sleep(3)
-
- profile_nr = 1
- profile = profiles[profile_nr]
-
- _print(profile.container, user.username, f"Börjar med profilen {profile.name}")
+ print('Mode:', mode)
+ print('Write:', write)
+
+ # Hämta en användare att kolla upp
+ user = User(str(userdoc['_key']).strip(), mode, other_pictures=url_other_pictures)
+
+ if "url_other_pictures" in globals():
+ l = []
+ for url in url_other_pictures:
+ l.append(url[url.find("facebook.com") + 12 :])
+ user.url_other_pictures = l
+
+ # Hämta profil
+ profile = new_profile(container, proxieservers)
+ profile.write = write
+
+ update_cookie(profile.browser.session.cookies, profile)
+ sleep(3)
# Gå igenom de användare som efterfrågats
- try:
- while True:
- for user in users:
- # Set för kollade bilder och kollade medlemmar
- all_pictures = set([doc["_key"] for doc in db.collection("pictures").all()])
- all_pictures_start = all_pictures.copy()
- members_checked = checked_members()
- profile.container = user.username
-
- # Hämta reaktioner för den första användaren
- if any([user.username not in members_checked, mode == 'force']):
- try:
- profile_picture_reactions(profile, user, all_pictures, first_user=True, mode=mode)
- except:
- _print(profile.container, user.username, traceback.format_exc())
- if len(users) == 1:
- for profile in profiles:
- profile.unused()
- friends = friends_of_user(user.username)
- friends_unchecked = list(set(friends) - set(members_checked))
-
- _print(profile.container, user.username, f"\nKlar med, {user.username}\n")
- _print(profile.container, user.username, f"Vänner som reagerat: {len(friends)}")
- _print(profile.container, user.username, "\nVänner att kolla:")
-
- for friend in friends_unchecked:
- print(friend)
- _print(profile.container, user.username, ', '.join([friend for friend in friends_unchecked]), silent=True)
- print()
-
- # Hämta reaktioner för users vänner (som reagerat)
+
+ if lookups == "leak_lookups":
+ id = user.username
+ check_profile_status(profile, user)
+ if profile.blocked:
+ profile = blocked_profile(profile, proxieservers=proxieservers)
+ profile.open(url_bas + "/" + user.username)
+ url = profile.browser.state.url.strip("/").strip("?_rdr")
+ if "php?" not in url:
+ user = User(str(url[url.rfind("/") + 1 :]).strip(), mode)
+ user.id = id
+ sleep_(4)
+ container = str(user.username)
+ profile.container = container
+
+ if "container" not in globals():
+ container = str(user.username)
+ profile.container = container
+
+ profile.users_checked += 1
+
+ # Hämta reaktioner för den första användaren
+ if any([not check_for_user(user.username, mode=mode), mode == "force"]):
+ try:
+ while True:
+ # Uppdatera in_use
+ profile.update_time()
+ profile = profile_picture_reactions(
+ profile, user, first_user=True, mode=mode
+ )
+ if profile.blocked:
+ profile = blocked_profile(profile, proxieservers=proxieservers)
+ else:
+ break
+ except:
+ _print(profile, user, traceback.format_exc())
+
+ if mode == 'solo':
+ exit()
+
+ friends = friends_of_user(user.username)
+ _print(profile, user, f"\nKlar med, {user.username}\n")
+ _print(profile, user, f"Vänner som reagerat: {len(friends)}")
+ _print(profile, user, "\nVänner att kolla:")
+
+ friends_unchecked = []
+ for friend in friends:
+ if not check_for_user(friend):
+ print(friend)
+ friends_unchecked.append(friend)
+
+ _print(profile, user, [friends_unchecked], silent=True)
+ _print(profile, user, f'Totalt: {len(friends_unchecked)}')
+ print()
+
+ # Hämta reaktioner för users vänner (som reagerat)
+ count_friends = 0
+ for friend in friends_unchecked:
+ if datetime.now().strftime("%H") == '03' and int(datetime.now().strftime("%M")) < 30: # Sov för att kunna säkerhetskopieraa
+ sleep(1800)
+ count_friends += 1
+ user = User(str(friend), mode, other_pictures=[])
+ sleep_(2)
+
+ # Uppdatera in_use
+ profile.update_time()
+ try:
+ if not check_for_user(user.username):
+ p = profile_picture_reactions(profile, user, mode=mode)
+ if isinstance(p, Profile):
+ profile = p
+
+ except Exception as e: # Fel4
+ write_error(
+ 4,
+ profile,
+ e=e,
+ user=user,
+ traceback=traceback.format_exc(),
+ soup=profile.viewing(),
+ )
+ _print(profile, user, f"\nFel: {str(user.username)}\n")
+ sleep_(15)
+
+ if not profile.blocked:
+ _print(profile, user, f"Klar med {user.username} \n")
+
+ # Rotera fb-profiler
+ if count_friends > 2 * mode_nr:
+ if random.randrange(0, 2, 1) == 1:
+ profile = new_profile(container, proxieservers=proxieservers)
+ count_friends = 0
+ _print(profile, user, f"Växlar till {profile.name}")
+ elif count_friends > 4 * mode_nr:
+ profile = new_profile(container, proxieservers=proxieservers)
count_friends = 0
- for friend in friends_unchecked:
- count_friends += 1
- user = User(str(friend), mode)
- sleep_(2)
-
- try:
- profile_picture_reactions(
- profile, user, all_pictures, mode=mode
- )
- except Exception as e: # Fel4
- write_error(
- 4,
- e=e,
- user=user.username,
- profile=profile.container,
- traceback=traceback.format_exc(),
- soup=profile.viewing(),
- )
- _print(profile.container, user.username, f"\nFel: {str(user.username)}\n")
- sleep_(15)
-
- if profile.blocked == False:
- _print(profile.container, user.username, f"Klar med {user.username} \n")
-
- # Rotera fb-profiler
- if count_friends > 5 * mode_nr:
- if random.randrange(0, 2, 1) == 1:
- profile_nr += 1
- count_friends = 0
- _print(profile.container, user.username, f"Växlar till {profiles[profile_nr].name}")
- elif count_friends > 9 * mode_nr:
- profile_nr += 1
- count_friends = 0
- _print(profile.container, user.username, f"Växlar till {profiles[profile_nr].name}")
-
- if profile_nr > len(profiles) - 1:
- profile_nr = 0
-
- elif profile.blocked == True:
- # Ta bort profilen ur databasen
- report_blocked(profile, users)
- remove_profile(profile.doc)
- # Ta bort från listan på fb-profiler som används
- profiles.remove(profile)
- # Försök lägga till en ny fb-profil (om det finns en skapad och ledig i databasen)
- try:
- doc = get_profile()
- profiles[profile_nr] = Profile(doc, container)
- _print(profile.container, user.username, f"Laddat ny profil: {profiles[profile_nr].name}")
- sleep(3)
- except e:
- _print(profile.container, user.username, "Det behövs nya profiler...")
- if len(profiles) == 0:
- break
- for s in range(0, 1600 / len(profiles)):
- print(user, f"Sover {600-s} sekunder till... ", end="\r")
- profile_nr += 1
- _print(profile.container, user.username, f"Försöker med {profiles[profile_nr].name}.")
-
- profile = profiles[profile_nr]
-
-
- except:
- finish()
-
\ No newline at end of file
+ _print(profile, user, f"Växlar till {profile.name}")
+
+ elif profile.blocked:
+ profile = blocked_profile(profile, proxieservers=proxieservers)
+
+ _print(profile, None, f"Klar med alla vänner.")
+
diff --git a/facebook/arangodb.py b/facebook/arangodb.py
index 5e24e6d..e1fa92a 100644
--- a/facebook/arangodb.py
+++ b/facebook/arangodb.py
@@ -1,95 +1,101 @@
from getpass import getpass
+from random import randint
from time import sleep
import json
+from datetime import datetime
+from json2html import json2html
-import nacl.secret
-import nacl.utils
from arango import ArangoClient
from config import *
-# Starta koppling till arangodb
-# Avkryptera lösen till arango
-try:
- # Om scriptet körs på Macbook finns löseordet i en fil
- with open('password_arango.txt') as f:
- pwd = f.readline()
-except FileNotFoundError:
- for i in range(0, 6, 1):
- if i == 5:
- exit()
- try:
- key = "sssladnnklja" + getpass()
- pwd = (
- nacl.secret.SecretBox(key.encode())
- .decrypt(pwd_arango, encoder=nacl.encoding.HexEncoder)
- .decode("utf-8")
- )
- break
- except:
- print("Fel lösenord.")
- sleep(1)
-db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)
+for i in range(0, 6, 1):
+ if i == 5:
+ exit()
+ try:
+ # Om scriptet körs på Macbook finns lösenordet i en fil
+ with open("../password_arango.txt") as f:
+ pwd = f.readline()
+ except FileNotFoundError:
+ if 'pwd' not in globals():
+ pwd = getpass(f'Lösenord för {user_arango}: ')
+
+ try:
+ db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)
+ db.collection('members').random() # För att testa löseordet/kopplingen.
+ break
+ except:
+ print("Fel lösenord.")
+ sleep(1)
-from helpers import now, _print
+from helpers import now, _print, nowstamp, sleep_
+from classes import Profile
def checked_members():
cursor = db.aql.execute(
- """
+ """
FOR doc IN members
FILTER doc.checked == true
RETURN doc._key
"""
- )
+ )
members_checked = set([doc for doc in cursor])
return members_checked
+def update_inuse(profile):
+ db.collection("profiles").update(profile["doc"]["id"])
+
+
def count_docs(col):
- cursor = db.aql.execute(
- """
+ cursor = db.aql.execute(
+ """
FOR doc IN @@col
COLLECT WITH COUNT INTO length
RETURN length
""",
- bind_vars={"@col": col}
- )
+ bind_vars={"@col": col},
+ )
return cursor.next()
-def report_blocked(profile, users):
- db.insert_document({
- '_id':f'reports/{now()}',
- 'profile': profile.name,
- 'users': [user.username for user in users],
- })
-
-def write_report(users, pictures):
- db.insert_document({
- '_id':f'reports/{now()}',
- 'users': [user.username for user in users],
- 'members': count_docs('members'),
- 'total_picture_reactions':count_docs('picture_reactions'),
- 'pictures':count_docs('pictures'),
- 'new_pictures': pictures
- })
+def report_blocked(profile):
+ try:
+ db.insert_document(
+ "reports",
+ {
+ "_key": str(profile.name).replace(' ', ''),
+ "profile": profile.__dict__
+ },
+ overwrite=True,
+ )
+ except:
+ _print(profile, profile.container, f'Kunde inte rapportera blockerad: {profile.name}.')
-def get_profile(created=True):
+def get_profile(db=db, collection='mullvad'):
""" Hämtar profil från profiles """
- cursor = db.aql.execute(
+
+
+ while True:
+ cursor = db.aql.execute(
"""
- FOR doc IN profiles
- FILTER doc.in_use == false
- FILTER doc.created == @created
- RETURN doc
- """,
- bind_vars={'created': created}
- )
- return cursor.next()
+ FOR doc IN @@col
+ FILTER doc.in_use < @inuse
+ RETURN doc
+ """,
+ bind_vars={"inuse": nowstamp() - 1200, '@col': f'profiles_{collection}'}
+ )
+ profiles = [profile for profile in cursor]
+ if profiles == []:
+ sleep(180)
+ else:
+ profile = profiles[randint(0, len(profiles) - 1)]
+ return profile
+
def friends_of_user(user):
"""Returnernar användare som reagerat på user:s bilder"""
@@ -104,28 +110,206 @@ def friends_of_user(user):
return [doc[8:] for doc in cursor]
-def remove_profile(profile):
- db.collection("profiles").delete(profile['_key'], silent=True, ignore_missing=True)
- _print(profile.container, f'{profile.name} blockerad och borttagen {now()}.'
+def remove_profile(profile, proxieservers='mullvad'):
+ """ Tar bort en blockerad profil från databasen. """
+ _print(profile, None, f"Tar bort {profile.name}.")
+
+ db.collection(f'profiles_{proxieservers}').delete(
+ profile.doc["_key"], silent=True, ignore_missing=True
)
+ _print(profile, profile.container, f"{profile.name} blockerad och borttagen {now()}.")
+
# TODO #2 Bättre funktion för backup av databasen
+def arango_connect(pwd):
+ return ArangoClient(hosts=host_arango).db(
+ db_arango, username=user_arango, password=pwd
+ )
+
+
+def check_for_user(username, mode=''):
+ """ Checks if a user exist in db and if it's checked """
+ checked = False
+ if db.collection("members").has(username):
+ member = db.collection('members').get(username)
+ if 'checked' in member:
+ if member['checked'] == True:
+ checked = True
+ if mode == 'all':
+ if 'mode' in member:
+ if member['mode'] in ['few', 'solo']:
+ checked = False
+ return checked
+
+
+def check_for_picture(id):
+ """ Checks if a picture exist in db """
+ return db.collection("pictures").has(id)
+
+
+def get_user(collection="lookups"):
+ """ Hämtar användare att kolla upp från lookups """
+
+ if collection == "leak_lookups":
+ doc = db.collection("leak_lookups").random()
+ doc["other"] = []
+ db.collection(collection).delete(doc["_key"])
+ else:
+ cursor = db.aql.execute(
+ """
+ FOR doc IN @@col
+ RETURN doc
+ """,
+ bind_vars={"@col": collection},
+ )
+ try:
+ doc = cursor.next()
+ if "other" not in doc:
+ doc["other"] = []
+ db.collection(collection).delete(doc["_key"])
+ except StopIteration:
+ doc = None
+
+ return doc
+
+
def backup(db):
- """Skapar en json-backup för specificerade collections.
+ """Skapar en json-backup och statistik för specificerade collections.
Args:
db: databaskoppling till aktuell databas
"""
- d = {}
- for col in ['members', 'pictures', 'picture_reactions', 'profiles']:
- l = []
- for doc in db.collection(col).all():
- l.append(doc)
- d[col] = l
- with open('data/backup.json', 'w') as f:
- json.dump(d, f)
- print(f'Senaste backup: {now()}')
+ while True:
+ if not datetime.now().strftime("%H") == '03' and int(datetime.now().strftime("%M")) < 10:
+ sleep(120)
+ continue
+ collections = ["members", "pictures", "picture_reactions", "profiles", "stats"]
+ for col in collections:
+ l = []
+ count = 0
+ icount = 0
+ for doc in db.collection(col).all():
+ count += 1
+ l.append(doc)
+ if count == 1000000:
+ icount += 1
+ count = 0
+ with open(f"data/backup_{col}_{icount}.json", "w") as f:
+ json.dump(l, f)
+ l = []
+ icount += 1
+ with open(f"data/backup_{col}_{icount}.json", "w") as f:
+ json.dump(l, f)
+ l = []
+ print(f"Senaste backup: {now()}")
+ write_stats()
+ sleep(82800)
+
+
+def write_stats(continuous=False):
+ while True:
+ d = {}
+ for col in db.collections():
+ if not col['system']:
+ d[col['name']] = db.collection(col['name']).count()
+ del d['stats']
+ #d['time'] = now()
+ cursor = db.aql.execute(
+ """
+ FOR doc IN members
+ FILTER doc.checked == true
+ COLLECT WITH COUNT INTO length
+ RETURN length
+ """
+ )
+ d['checked_members'] = cursor.next()
+
+
+ # Hur många konton per säljare som finns kvar
+ cursor = db.aql.execute(
+ '''
+ for doc in profiles
+ filter has(doc, "vendor")
+ COLLECT vendor = doc.vendor WITH COUNT INTO length
+ RETURN {
+ "vendor" : vendor,
+ "active" : length
+ }
+ ''')
+ d['active_vendors'] = [doc for doc in cursor]
+
+ d['_key'] = now()[:13]
+ db.insert_document( "stats", d, overwrite=True)
+ # Skriv en html-fil
+ with open('webbapp/templates/stats.html', 'a+') as html:
+ html.truncate(0)
+ html.write('
')
+
+ html.write(json2html.convert(json = d))
+
+ # Sov för att fortsätta senare
+ if continuous:
+ sleep(86400)
+ else:
+ break
+
+def blocked_profile(profile, proxieservers):
+ """ Tar bort profilen som blivit blockad och returnerar en ny. """
+ _print(profile, None, f'Rapporterar att {profile.name} blockats.')
+ report_blocked(profile)
+ _print(profile, None, f'Tar bort {profile.name} från databasen.')
+ remove_profile(profile, proxieservers)
+ _print(profile, None, f'Hämtar en ny profil.')
+ profile = new_profile(profile.container, proxieservers)
+ return profile
+
+
+def new_profile(container, proxieservers):
+ """ Hämtar en ny profil. """
+ profile = Profile(get_profile(proxieservers=proxieservers), container, proxieservers)
+ _print(profile, None, f'Hämtade profilen {profile.name}. Login = {profile.logged_in}.')
+ if profile.logged_in == False:
+ profile.accept_cookies()
+ sleep_(2)
+ profile.login()
+ sleep_(2)
+ try:
+ profile.open(url_bas)
+ if "accept all" in profile.viewing().text.lower():
+ _print(profile, None, f'Accepterar cookies {profile.name}.')
+ profile.accept_cookies()
+ sleep_(3)
+ except:
+ pass
+ return profile
+
+def find_id():
+ "https://mbasic.facebook.com/leif.jonsson.98499/about?lst=100064897389168%3A100000134933241%3A1615858816"
+
+ cursor = db.aql.execute(
+ """
+ for doc in members
+ filter has(doc, "about")
+ filter doc.facebook_id == ''
+ filter doc.about != false
+ return doc
+ """,
+ )
+
+ n = 0
+ for doc in cursor:
+ about = doc['about']
+ try:
+ doc['facebook_id'] = about[about.find('%')+3: about.rfind('%')]
+ db.update_document(doc, silent=True, check_rev=False)
+ #sleep(0.01)
+ n += 1
+ print(n, end = '\r')
+ except AttributeError:
+ pass
+
+
db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)
diff --git a/facebook/classes.py b/facebook/classes.py
index b69df6d..fddc632 100644
--- a/facebook/classes.py
+++ b/facebook/classes.py
@@ -1,6 +1,6 @@
-import pickle
import random
from datetime import datetime
+from time import sleep
import requests
import werkzeug
@@ -10,26 +10,31 @@ werkzeug.cached_property = werkzeug.utils.cached_property
from robobrowser import RoboBrowser
from arangodb import db
-from config import *
-from helpers import sleep_, update_cookie
+from config import url_bas
+from helpers import sleep_, update_cookie, write_error, nowstamp, _print
class User:
- def __init__(self, username, mode):
+ def __init__(self, username, mode, other_pictures=[]):
self.collection = "members"
self.username = str(username)
self.mode = mode
self.fetched = datetime.now().strftime("%Y%m%d")
- self.url_coverphotos = ''
- self.id = ''
- self.url_likes = ''
- self.url_about = ''
- self.url_timeline = ''
- self.profile_pictures = ''
- self.url = ''
- self.name = ''
- self.url_other_picture = ''
+ self.url_coverphotos = ""
+ self.id = ""
+ self.url_likes = ""
+ self.url_about = ""
+ self.url_timeline = ""
+ self.url_profilepictures = ""
+ self.profile_pictures = 0
+ self.pictures = []
+ self.url_friends = ""
+ self.url = ""
+ self.name = ""
+ self.url_other_pictures = other_pictures
self.reactions = 0
+ self.profile_pictures = 0
+ self.checked_pictures = []
def add_to_db(self):
# Lägg till profilen till arrango
@@ -47,34 +52,38 @@ class User:
"cover photos": self.url_coverphotos,
"fetched": self.fetched,
"reactions": self.reactions,
- 'mode': self.mode
+ "mode": self.mode,
+ "pictures": self.pictures,
},
overwrite_mode="update",
- silent=True,
- keep_none=False
+ silent=True,
+ keep_none=False,
)
def checked(self):
db.update_document(
{
- "_id": "members/" + str(self.username),
- "checked": True,
- "pictures_checked": self.profile_pictures,
- "reaction": self.reactions
- })
-
+ "_id": "members/" + str(self.username),
+ "checked": True,
+ "pictures_checked": len(self.checked_pictures),
+ "checked_pictures": self.checked_pictures,
+ "reactions": self.reactions,
+ }
+ )
class Picture:
def __init__(self, user):
self.collection = "pictures"
self.user = user
- self.id = ''
- self.url_full = ''
- self.date = ''
- self.url = ''
- self.no_reactions = ''
+ self.id = ""
+ self.url_full = ""
+ self.date = ""
+ self.url = ""
+ self.no_reactions = ""
self.reactions = []
+ self.src = ""
+
def add_to_db(self):
db.insert_document(
@@ -86,80 +95,128 @@ class Picture:
"url": self.url,
"no_reactions": self.no_reactions,
"user": self.user,
+ "src": self.src,
},
overwrite_mode="update",
- silent=True,
- keep_none=False
+ silent=True,
+ keep_none=False,
)
+
class Profile:
- def __init__(self, profile, container):
- """ Creates a new profile to do searches with.
+ def __init__(self, profile, container, proxieservers):
+ """Creates a new profile to do searches with.
Args:
profile (dict): Document fetched from database.
- container (string): Container name.
- """
+ container (str): Docker container that runs the script.
+ """
- # Uppdatera dokumentet i arango
self.doc = profile
- self.doc['in_use'] = True
- db.update_document(self.doc, check_rev=False)
# Användaruppgifter
self.name = self.doc["name"].strip()
self.email = self.doc["email"]
self.pwd = self.doc["pwd"]
self.server = self.doc["server"]
-
+ self.cookie = self.doc["cookie"]
+ self.useragent = self.doc["useragent"]
+
+ self.proxieservers = proxieservers
self.blocked = False
- self.container = container
+ self.container = str(container)
+ self.users_checked = 0
+ self.write = True
# Ange proxies
session = requests.Session()
- session.proxies = {
- "https": "socks5://'8155249667566524'@{}".format(self.server),
- "http": "socks5://'8155249667566524'@{}".format(self.server),
- }
+ session.proxies = self.doc["proxies"]
# Starta browser
- user_agent = "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e Safari/602.1"
+ user_agent = self.useragent
self.browser = RoboBrowser(
session=session, user_agent=user_agent, history=False, parser="lxml"
)
+
+ # TODO Ta bort gamla metoden om nya (hämta från doc) fungerar
+ # try:
+ # # Försök hämta cookie från fil
+ # self.browser.session.cookies = pickle.load(
+ # open("data/cookie_{}.pkl".format(self.name), "rb")
+ # )
+ # self.logged_in = True
+
try:
- self.browser.session.cookies = pickle.load(
- open("data/cookie_{}.pkl".format(self.name), "rb")
- )
+ self.browser.session.cookies.update(self.cookie)
self.logged_in = True
except:
self.logged_in = False
+ def update_time(self):
+ """Uppdatera dokumentet i arango."""
+ self.doc["in_use"] = nowstamp()
+ db.update_document(self.doc, check_rev=False)
+
def viewing(self):
- """ Returnerar browser i html-format """
+ """Returnerar browser i html-format"""
return self.browser.parsed
-
+
+ def open(self, url):
+ n = 0
+ while True:
+ n += 1
+ sleep(1)
+ try:
+ # Försök öppna url, om det misslyckas så vänta lite och försök sen igen
+ self.browser.open(url)
+ if "/a/nux/wizard/nav.php?step=phone&skip" in self.viewing():
+ self.browser.open(
+ url_bas + "/a/nux/wizard/nav.php?step=phone&skip"
+ )
+ break
+ except Exception as e:
+ print(e)
+ print(n)
+ _print(self, None, f"Kunde inte öppna url {url}")
+ if n == 5:
+ if "Connection refused" in e:
+ self.doc["e"] = e
+ db.insert_document("blocked_profiles", self.doc)
+ n = 0
+ from arangodb import get_profile, remove_profile
+
+ # Ta bort den gamla profilen från databasen och ersätt profile med nytt innehåll från ny profil
+ remove_profile(self)
+ self.__init__(get_profile(self.proxieservers), self.container)
+ _print(self, None, f"Ny profil hämtad {self.email}")
+ self.update_time()
+ else:
+ sleep(40)
def accept_cookies(self):
- """ Accepterar cookies """
+ """Accepterar cookies"""
self.browser.open("https://mbasic.facebook.com")
soup = BeautifulSoup(str(self.browser.parsed), "lxml")
- if 'accept all' not in soup.text.lower():
+ if "accept all" not in soup.text.lower():
sleep_(2)
- cookie_accept_url = "https://mbasic.facebook.com/cookie/consent-page"
+ cookie_accept_url = "https://mbasic.facebook.com/cookie/consent-page"
self.browser.open(cookie_accept_url)
sleep_(2)
try:
form = self.browser.get_form()
self.browser.submit_form(form)
- print(f"Accepterade cookies för {self.name}")
+ _print(self, None, f"Accepterade cookies för {self.name}")
sleep_(2)
- update_cookie(self.browser.session.cookies, self.name)
- except Exception as e:
- print(f"Accepterade inte cookies för {self.name}")
-
+ update_cookie(self.browser.session.cookies, self)
+ except:
+ try:
+ write_error(12, self, soup=self.browser.parsed)
+ except:
+ pass
+ _print(self, None, f"Accepterade inte cookies för {self.name}")
+
def login(self):
- """ Loggar in på Facebook. """
+ """Loggar in på Facebook."""
print("Loggar in {}".format(self.name))
@@ -168,49 +225,57 @@ class Profile:
# Kolla om browser redan är inloggad
soup = BeautifulSoup(str(self.browser.parsed), "lxml")
- if 'log out' in soup.text.lower():
+ if "log out" in soup.text.lower():
print("Redan inloggad.")
-
- # Hitta och fyll i formulär
- form = self.browser.get_form(id="login_form")
- form["email"].value = self.email
- form["pass"].value = self.pwd
- self.browser.submit_form(form, submit=form["login"])
- # Vänta lite och uppdatera cookie
- print("Loggade in.")
- sleep_(2)
+ try:
+ # Hitta och fyll i formulär
+ form = self.browser.get_form(id="login_form")
+ form["email"].value = self.email
+ form["pass"].value = self.pwd
+ self.browser.submit_form(form, submit=form["login"])
+ # Vänta lite och uppdatera cookie
+ print("Loggade in.")
+ sleep_(2)
+ self.open(url_bas)
+ sleep_(2)
+ except TypeError:
+ try:
+ write_error(11, self, soup=soup, profile=self.name)
+ except:
+ pass
+
+ def update_cookie(self, cookie):
+ self.cookie = cookie
+ db.update_document({"_id": self.doc["_id"], "cookie": cookie}, check_rev=False)
- def unused(self):
- """ Sätter user till False för valda profiler """
- self.doc["in_use"] = False
- db.update_document(self.doc, silent=True)
class Proxies:
def __init__(self):
self.proxies = [
- 'gb25-wg.socks5.mullvad.net:1080',
- 'gb26-wg.socks5.mullvad.net:1080',
- 'gb27-wg.socks5.mullvad.net:1080',
- 'gb28-wg.socks5.mullvad.net:1080',
- 'gb29-wg.socks5.mullvad.net:1080'
- ]
+ "gb25-wg.socks5.mullvad.net:1080",
+ "gb26-wg.socks5.mullvad.net:1080",
+ "gb27-wg.socks5.mullvad.net:1080",
+ "gb28-wg.socks5.mullvad.net:1080",
+ "gb29-wg.socks5.mullvad.net:1080",
+ ]
+
def get_proxie(self):
return self.proxies.pop(random.randrange(0, len(self.proxies), 1))
+
class Friend:
def __init__(self, user):
self.collection = "members"
self.user = user # The friends friend
- self.username = ''
- self.url = ''
- self.name = ''
- self.single = ''
+ self.username = ""
+ self.url = ""
+ self.name = ""
def add_to_db(self):
db.insert_document(
self.collection,
{
- "_key": self.username,
+ "_key": str(self.username),
"url": url_bas + self.url,
"name": self.name,
},
diff --git a/facebook/config.py b/facebook/config.py
index da2f82e..c84e19e 100644
--- a/facebook/config.py
+++ b/facebook/config.py
@@ -1,9 +1,18 @@
+from getpass import getpass
+def set_pwd(_pwd=None):
+ global pwd
+ if _pwd == None:
+ _pwd = getpass('Lösenord för Arango-användaren:')
+ pwd = _pwd
# Info för arangodb
user_arango = "Lasse"
pwd_arango = "4c071768bedc259288361c07aafd8535fca546086fada4e7b5de4e2bb26b0e70fa8d348c998b90d032a5b8f3fdbae1881b843021e3475198e6fb45f58d8dc450bd52f77d"
db_arango = "facebook"
-host_arango = "http://arango.lasseedfast.se"
+host_arango = 'http://192.168.0.4:8529'
+#host_arango = "http://arango.lasseedfast.se"
# Andra uppgifter
-url_bas = "https://mbasic.facebook.com"
\ No newline at end of file
+url_bas = "https://mbasic.facebook.com"
+user_agent = "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e Safari/602.1"
+mullvad = '8155249667566524'
\ No newline at end of file
diff --git a/facebook/face_enc b/facebook/face_enc
new file mode 100644
index 0000000..21d83f4
Binary files /dev/null and b/facebook/face_enc differ
diff --git a/facebook/faces.py b/facebook/faces.py
new file mode 100644
index 0000000..e7ef894
--- /dev/null
+++ b/facebook/faces.py
@@ -0,0 +1,110 @@
+import os
+import pickle
+import time
+
+import cv2
+import face_recognition
+
+
+def build_data():
+ """ Build the face_enc file with data to recognize from """
+
+ knownEncodings = []
+ knownNames = []
+
+ members = os.listdir('../profile_pictures')
+
+ #get paths of each file in folder named Images
+ #Images here contains my data(folders of various persons)
+ for member in members:
+ if '.DS_Store' in member:
+ continue
+ imagePaths = []
+ for path in os.listdir(f'../profile_pictures/{member}'):
+ if '.jpg' in path:
+ imagePaths.append(f'../profile_pictures/{member}/{path}')
+
+ # loop over the image paths
+ for imagePath in imagePaths:
+ print(imagePath)
+ # load the input image and convert it from BGR (OpenCV ordering)
+ # to dlib ordering (RGB)
+ image = cv2.imread(imagePath)
+ rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+ #Use Face_recognition to locate faces
+ boxes = face_recognition.face_locations(rgb, number_of_times_to_upsample = 2) #,model='hog'
+ # compute the facial embedding for the face
+ encodings = face_recognition.face_encodings(image, boxes)
+ # loop over the encodings
+ for encoding in encodings:
+ knownEncodings.append(encoding)
+ knownNames.append(member)
+ #save emcodings along with their names in dictionary data
+ data = {"encodings": knownEncodings, "names": knownNames}
+ #use pickle to save data into a file for later use
+ with open("face_enc", "wb") as f:
+ f.write(pickle.dumps(data))
+ f.close()
+
+def identify_face(imagePath):
+ #find path of xml file containing haarcascade file
+ cascPathface = os.path.dirname(
+ cv2.__file__) + "/data/haarcascade_frontalface_alt2.xml"
+ # load the harcaascade in the cascade classifier
+ faceCascade = cv2.CascadeClassifier(cascPathface)
+ # load the known faces and embeddings saved in last file
+ data = pickle.loads(open('face_enc', "rb").read())
+ #Find path to the image you want to detect face and pass it here
+ image = cv2.imread(imagePath)
+ rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+ #convert image to Greyscale for haarcascade
+ gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+ faces = faceCascade.detectMultiScale(gray,
+ scaleFactor=1.1,
+ minNeighbors=5,
+ minSize=(60, 60),
+ flags=cv2.CASCADE_SCALE_IMAGE)
+
+ # the facial embeddings for face in input
+ encodings = face_recognition.face_encodings(rgb)
+ names = []
+ # loop over the facial embeddings incase
+ # we have multiple embeddings for multiple fcaes
+ for encoding in encodings:
+ #Compare encodings with encodings in data["encodings"]
+ #Matches contain array with boolean values and True for the embeddings it matches closely
+ #and False for rest
+ matches = face_recognition.compare_faces(data["encodings"],
+ encoding)
+ #set name =unknown if no encoding matches
+ name = "Unknown"
+ # check to see if we have found a match
+ if True in matches:
+ #Find positions at which we get True and store them
+ matchedIdxs = [i for (i, b) in enumerate(matches) if b]
+ counts = {}
+ # loop over the matched indexes and maintain a count for
+ # each recognized face face
+ for i in matchedIdxs:
+ #Check the names at respective indexes we stored in matchedIdxs
+ name = data["names"][i]
+ #increase count for the name we got
+ counts[name] = counts.get(name, 0) + 1
+ #set name which has highest count
+ name = max(counts, key=counts.get)
+ print(counts)
+
+ # update the list of names
+ names.append(name)
+ # loop over the recognized faces
+ for ((x, y, w, h), name) in zip(faces, names):
+ # rescale the face coordinates
+ # draw the predicted face name on the image
+ cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
+ cv2.putText(image, name, (x, y), cv2.FONT_HERSHEY_SIMPLEX,
+ 0.75, (0, 255, 0), 2)
+ cv2.imshow("Frame", image)
+ cv2.waitKey(0)
+
+
+identify_face('/Users/Lasse/Datorgemensamt/Programmeringsprojekt/Facebook/fb-scraper/profile_pictures/millington.jiang/4138068259557849.jpg')
\ No newline at end of file
diff --git a/facebook/gephi (kopia).py b/facebook/gephi (kopia).py
new file mode 100644
index 0000000..45586f4
--- /dev/null
+++ b/facebook/gephi (kopia).py
@@ -0,0 +1,185 @@
+import locale
+import re
+from datetime import datetime
+
+import networkx as nx
+import pandas as pd
+from numpy.core.numeric import NaN
+
+locale.setlocale(locale.LC_TIME, "en_US")
+
+from arangodb import db
+
+
+def nodes_from_list(
+ nodes, collection="members", return_fields="{'_key': doc._key, 'name': doc.name}"
+):
+ aql_edges = f"""
+ FOR doc IN @@ecollection
+ FILTER doc._id IN @nodes
+ RETURN {return_fields}
+ """
+ cursor = db.aql.execute(
+ aql_edges, bind_vars={"@ecollection": collection, "nodes": nodes}
+ )
+ return [doc for doc in cursor]
+
+
+def edges_from_nodes(
+ nodes, edge_collections=["picture_reactions"], simple=True, mode="or"
+):
+ """
+ Returnerar en df med relationer för valda noder och relationtabeller.
+
+ Args:
+ nodes (list): Noder som ska ingå i relationerna
+ edge_collections (list, optional): Relationtabeller att hämta relationer från. Defaults to ['messages'].
+ simple (bool, optional): Simple ger bara _from, _to och _key. Defaults to True.
+
+ Returns:
+ pd.DataFrame: DataFrame.
+ """
+
+ if simple:
+ return_fields = (
+ "{'_to': doc._to, '_from': doc._from, '_id':doc._id, '_key':doc._key}"
+ )
+ else:
+ return_fields = "doc"
+
+ edges = []
+
+ for collection in edge_collections:
+ aql = f"""
+ FOR doc IN @@edge_collection
+ FILTER doc._from IN @nodes {mode} doc._to IN @nodes
+ RETURN {return_fields}
+ """
+ cursor = db.aql.execute(
+ aql,
+ bind_vars={
+ "@edge_collection": collection,
+ "nodes": nodes,
+ },
+ )
+
+ edges = edges + [doc for doc in cursor]
+
+ return edges
+
+
+def convert_date(date):
+ try:
+ new_date = datetime.strptime(date, "%d %b %Y")
+ except ValueError:
+ try:
+ new_date = datetime.strptime(date, "%d %B %Y")
+ except ValueError:
+ try:
+ new_date = datetime.strptime(date, "%b %d, %Y")
+ except ValueError:
+ try:
+ new_date = datetime.strptime(date, "%B %d, %Y")
+ except ValueError:
+ try:
+ new_date = datetime.strptime(date + " 2021", "%d %b %Y")
+ except ValueError:
+ return ""
+ return new_date.strftime("%Y-%d-%d")
+ # return f'{new_date.date().year}-{new_date.date().month}-{new_date.date().day}'
+
+
+def export_network(members, n=2):
+ """ Exporterar en gexf-fil med noder utifrån en lista med medlemmar. """
+
+ filename = f"data/-.join({members}).-old.gexf"
+ ids = []
+ for member in members:
+ ids.append(f"members/{member}")
+ friends = set()
+
+ # Hämta relationer kopplade till members från databasen
+ edges = edges_from_nodes(ids)
+ for edge in edges:
+ friends.add(edge["_from"])
+ friends.add(edge["_to"])
+ edges = edges_from_nodes(list(friends))
+
+ # Skapa en dict där det syns vem som har interagerat med hur många
+ d = {}
+ for i in edges:
+ _to = i["_to"]
+ _from = i["_from"]
+ if _to not in d:
+ d[_to] = set([i["_from"]])
+ else:
+ d[_to] = d[_to] | set([i["_from"]])
+
+ if _from not in d:
+ d[_from] = set([i["_to"]])
+ else:
+ d[_from] = d[_from] | set([i["_to"]])
+
+ # Sålla ut så bara medlemmar som reagerat med [n] två av grundanvändarens vänner kommer med
+ friends = set(friends)
+ members = []
+ for key, value in d.items():
+ if len(value & friends) >= n or key in friends:
+ members.append(key)
+
+ # Skapa df med edges
+ edges = pd.DataFrame(
+ edges_from_nodes(members, mode="and", simple=False),
+ columns=["_key", "_to", "_from", "reaction", "picture"],
+ )
+ edges.set_index("_key", inplace=True)
+
+ # En lista på användare att ta med till nätverket
+ members = list(set(edges["_from"].unique()) | set(edges["_to"].unique()))
+
+ # Skapa noder till nätverket
+ nodes = nodes_from_list(
+ members
+ ) # , return_fields="{'id':doc._key, 'label':doc.name")
+ nodes = [(i["_key"], i) for i in nodes]
+
+ # Lägg till några kolumner i edges-tabellen
+ edges._from = edges._from.apply(lambda x: x[8:])
+ edges._to = edges._to.apply(lambda x: x[8:])
+ edges.picture = edges.picture.apply(
+ lambda x: re.search("\d+", x).group()
+ ) # Rensa bort url-info i de fall bilden har fått fel id
+
+ # Hämta bilder för att kunna lägga datum till edges
+ p = ["pictures/" + i for i in edges.picture.unique().tolist()]
+ d = {}
+ pictures = nodes_from_list(
+ p, collection="pictures", return_fields="{'id': doc._key, 'date':doc.date}"
+ )
+ for picture in pictures:
+ d[picture["id"]] = convert_date(picture["date"])
+
+ edges["date"] = edges.picture.apply(lambda x: d[x])
+
+ # Skapa graf utifrån relationer
+ G = nx.from_pandas_edgelist(
+ edges,
+ source="_from",
+ target="_to",
+ edge_attr=["reaction", "date"], #, "now"
+ create_using=nx.MultiDiGraph,
+ )
+
+ # Lägg till noderna i grafen
+ G.add_nodes_from(nodes)
+
+ # Exportera till filer
+ nx.write_gexf(
+ G,
+ filename
+ )
+
+
+if __name__ == "__main__":
+ export_network(["linda.kakuli"])
+ # export_network(input('Member: '))
diff --git a/facebook/gephi.py b/facebook/gephi.py
index 88c8487..1b0c704 100644
--- a/facebook/gephi.py
+++ b/facebook/gephi.py
@@ -1,6 +1,7 @@
import locale
import re
from datetime import datetime
+from sys import argv
import networkx as nx
import pandas as pd
@@ -25,42 +26,36 @@ def nodes_from_list(
return [doc for doc in cursor]
+
def edges_from_nodes(
- nodes, edge_collections=["picture_reactions"], simple=True, mode="or"
+ nodes, edge_collections=["picture_reactions"], mode="or"
):
"""
- Returnerar en df med relationer för valda noder och relationtabeller.
+ Returnerar en dict med relationer för valda noder och relationtabeller.
Args:
nodes (list): Noder som ska ingå i relationerna
edge_collections (list, optional): Relationtabeller att hämta relationer från. Defaults to ['messages'].
- simple (bool, optional): Simple ger bara _from, _to och _key. Defaults to True.
Returns:
- pd.DataFrame: DataFrame.
+ dict: Dict med relationer
"""
- if simple:
- return_fields = (
- "{'_to': doc._to, '_from': doc._from, '_id':doc._id, '_key':doc._key}"
- )
- else:
- return_fields = "doc"
-
+
edges = []
for collection in edge_collections:
aql_edges = f"""
FOR doc IN @@edge_collection
FILTER doc._from IN @nodes {mode} doc._to IN @nodes
- RETURN {return_fields}
+ RETURN doc
"""
cursor = db.aql.execute(
aql_edges,
bind_vars={
"@edge_collection": collection,
"nodes": nodes,
- },
+ }, stream=True
)
edges = edges + [doc for doc in cursor]
@@ -89,17 +84,32 @@ def convert_date(date):
# return f'{new_date.date().year}-{new_date.date().month}-{new_date.date().day}'
-def export_network(member, n=2):
- """ Exporterar en gexf-fil med noder utifrån en medlem. """
+def get_edges(member, n=2, lookups=[], common=True):
+ """ Returnerar en df med edges för vald member.
+
+ Args:
+ member (str): Username for member.
+ lookups (list): Användare att hitta gemensamt nätverk för
+ noncommon (bool): Om den ena användarens förstakontakter ska räknas till den andra användarens nätverk
+ Returns:
+ df: Dataframe with edges
+ """
member = f"members/{member}"
+ lookups = [f"members/{i}" for i in lookups]
member_friends = set()
-
# Hämta relationer kopplade till member från databasen
for edge in edges_from_nodes([member]):
member_friends.add(edge["_from"])
member_friends.add(edge["_to"])
- edges = edges_from_nodes(list(member_friends))
+ member_friends = list(member_friends)
+
+ if not common:
+ # Ta bort de andra i lookups så inte de får kompisars kompisar
+ member_friends = [friend for friend in member_friends if friend not in lookups] # ! Ska den här vara kvar?
+ for member in lookups:
+ member_friends.append(member)
+ edges = edges_from_nodes(member_friends)
# Skapa en dict där det syns vem som har interagerat med hur många
d = {}
@@ -120,34 +130,51 @@ def export_network(member, n=2):
member_friends = set(member_friends)
members = []
for key, value in d.items():
- if len(value & member_friends) >= n:
+ if len(value & member_friends) >= n or key in member_friends:
members.append(key)
# Skapa df med edges
edges = pd.DataFrame(
- edges_from_nodes(members, mode="and", simple=False),
+ edges_from_nodes(members, mode="and"),
columns=["_key", "_to", "_from", "reaction", "picture"],
)
edges.set_index("_key", inplace=True)
- # En lista på användare att ta med till nätverket
- members = list(set(edges["_from"].unique()) | set(edges["_to"].unique()))
+ return edges
+
- # Skapa noder till nätverket
- nodes = nodes_from_list(
- members
- ) # , return_fields="{'id':doc._key, 'label':doc.name")
- nodes = [(i["_key"], i) for i in nodes]
+def members_from_edges(edges):
+ """ En lista på användare att ta med till nätverket.
+ Args:
+ edges (df): Dataframe with edges.
+
+ Returns:
+ list: List of unique members in edges (to and from).
+ """
+
+ return list(set(edges["_from"].unique()) | set(edges["_to"].unique()))
+
+
+def edges_for_network(edges):
+ """ Prepare edges for the network
+
+ Args:
+ edges (df): Dataframe with edges
+
+ Returns:
+ df: Dataframe with edges prepared for network.
+ """
# Lägg till några kolumner i edges-tabellen
- edges._from = edges._from.apply(lambda x: x[8:])
- edges._to = edges._to.apply(lambda x: x[8:])
+ edges._from = edges._from.apply(lambda x: x[8:]) # Ta bort "members/"
+ edges._to = edges._to.apply(lambda x: x[8:]) # Ta bort "members/"
edges.picture = edges.picture.apply(
lambda x: re.search("\d+", x).group()
) # Rensa bort url-info i de fall bilden har fått fel id
# Hämta bilder för att kunna lägga datum till edges
p = ["pictures/" + i for i in edges.picture.unique().tolist()]
+
d = {}
pictures = nodes_from_list(
p, collection="pictures", return_fields="{'id': doc._key, 'date':doc.date}"
@@ -157,25 +184,145 @@ def export_network(member, n=2):
edges["date"] = edges.picture.apply(lambda x: d[x])
+ return edges
+
+
+def export_network(member):
+ """ Exporterar en gexf-fil med noder utifrån en medlem. """
+ filename = f"data/{member}_.gexf"
+
+ edges = get_edges(member, n=3)
+ members = members_from_edges(edges)
+
# Skapa graf utifrån relationer
G = nx.from_pandas_edgelist(
- edges,
+ edges_for_network(edges),
source="_from",
target="_to",
- edge_attr=["reaction", "date", "now"],
+ edge_attr=["reaction", "date"], #, "now"
create_using=nx.MultiDiGraph,
)
+ ## Skapa noder till nätverket
+ nodes = nodes_from_list(
+ members
+ ) # , return_fields="{'id':doc._key, 'label':doc.name")
+
# Lägg till noderna i grafen
- G.add_nodes_from(nodes)
+ G.add_nodes_from([(i["_key"], i) for i in nodes])
# Exportera till filer
nx.write_gexf(
G,
- f"data/network_test.gexf",
+ filename
)
+def common_friends(d, n=2):
+ """ Filtrera ut gemensamma vänner """
+
+ common_friends = {}
+ for _, value in d.items():
+ for friend in set(value):
+ if friend not in common_friends:
+ common_friends[friend] = 1
+ else:
+ common_friends[friend] += 1
+
+ l = []
+ for key, value in common_friends.items():
+ if value >= n:
+ l.append(key)
+
+ if l == []:
+ print('Inga gemensamma i nätverken.')
+ exit()
+
+ return l
+
+
if __name__ == "__main__":
- export_network("maria.hansson.botin")
+
+ lookups = [
+ 'katherine.zimmerman.754',
+ 'boogiesaman.bakhtiari',
+ 'lena.tidestromsagstrom',
+ 'bibi.rodoo',
+ 'mina.benaissa',
+ 'henrik.johnsson.73',
+ 'fabian.asserback',
+ '100005696055822',
+ 'fia.wiren',
+ 'daniel.kjellander.5'
+ ]
+
+ print('Samlar data för:')
+ for i in lookups:
+ print(i)
+ print(f'({len(lookups)} stycken\n')
+
+ # Hur många vänners vänners ska känna
+ if len(lookups) == 1:
+ n = 1
+ elif len(argv) > 1:
+ n = int(argv[1])
+ else:
+ #from math import sqrt
+ n = round(len(lookups)/2.2 + 1)
+ print(f'n = {n}')
+
+ if len(lookups) <= 3:
+ filename = f"../data/{'-'.join(lookups).replace('.','')}.gexf"
+ else:
+ from datetime import datetime
+ filename = f"../data/{datetime.now()}.gexf"
+
+ if len (lookups) == 1:
+ export_network(lookups[0])
+ exit()
+
+ d = {}
+ for member in lookups:
+ edges = get_edges(member, lookups=lookups, common = False, n=n)
+ friends = members_from_edges(edges)
+ d[member] = friends
+ print(member, len(friends))
+
+ # Filtrera gemensamma vänner
+ common = common_friends(d)
+
+ print('Common friends: ', len(common))
+
+ edges = pd.DataFrame(edges_from_nodes(common, mode='and')) # and om båda noderna ska vara med i common friends, annars or
+
+ members = members_from_edges(edges)
+
+ edges = edges_for_network(edges)
+ # Skapa graf utifrån relationer
+ G = nx.from_pandas_edgelist(
+ edges,
+ source="_from",
+ target="_to",
+ edge_attr=["reaction", "date"], #, "now"
+ create_using=nx.MultiDiGraph,
+ )
+
+ ## Skapa noder till nätverket
+ nodes = nodes_from_list(
+ members
+ ) # , return_fields="{'id':doc._key, 'label':doc.name")
+
+ # Lägg till noderna i grafen
+ G.add_nodes_from([(i["_key"], i) for i in nodes])
+
+ # Exportera till filer
+ nx.write_gexf(
+ G,
+ filename
+ )
+
+
+
+
+ #export_network("asifasghar")
# export_network(input('Member: '))
diff --git a/facebook/helpers.py b/facebook/helpers.py
index 5c896b6..0103bb5 100644
--- a/facebook/helpers.py
+++ b/facebook/helpers.py
@@ -2,8 +2,9 @@ import pickle
import random
from datetime import datetime
from time import sleep
+import json
-from arangodb import db
+from config import url_bas
def sleep_(t):
@@ -12,7 +13,7 @@ def sleep_(t):
"""
variation = 4 # Testa olika sovlängder för att inte få användaren blockerad
sleep(t * variation * random.randrange(85, 115, 1) / 100)
- if random.randrange(0, 60, 1) == 1:
+ if random.randrange(0, 50, 1) == 1:
longsleep = random.randrange(200, 300)
print('')
for s in range(0, longsleep):
@@ -21,14 +22,17 @@ def sleep_(t):
print()
sleep(random.randrange(0, 10, 1) / 4)
+
# TODO #1 spara cookies till db
-def update_cookie(cookies, profile_name):
+def update_cookie(cookies, profile):
""" Uppdaterar cookie för browser """
- with open("data/cookie_{}.pkl".format(profile_name), "wb") as f:
- pickle.dump(cookies, f)
+ # with open("data/cookie_{}.pkl".format(profile.name), "wb") as f:
+ # pickle.dump(cookies, f)
+ # cookies_dict = json.dumps(dict(cookies))
+ profile.update_cookie(cookies.get_dict())
-def write_error(nr, e="", traceback="", soup="", user="", url="", url_name="", profile=""):
+def write_error(nr, profile, e=" ", traceback="", soup="", user="", url="", url_name=""):
"""Skriver info efter error till arango
Args:
@@ -36,19 +40,27 @@ def write_error(nr, e="", traceback="", soup="", user="", url="", url_name="", p
e (str, optional): error. Defaults to "".
traceback (str, optional): The traceback from traceback.format_exc(). Defaults to "".
soup (str, optional): Soup. Defaults to "".
- user (str, optional): The user. Defaults to "".
+ user (class, optional): The user. Defaults to "".
url (str, optional): Url, if any. Defaults to "".
count (int, optional): Count, if any. Defaults to 0.
url_name (str, optional): The description of the url, if any. Defaults to "".
+ profile (user, optional): The profile.
"""
if url == "":
url = "ingen url"
url_name = "ingen url"
-
- # BARA VID FELSÖKNING
- _print(profile.container, e)
- _print(profile.container, traceback)
-
+ # try:
+ # # BARA VID FELSÖKNING
+ # print(profile, user, e)
+ # print(profile, user, traceback.format_exc())
+ # _print(profile, user, e)
+ # _print(profile, user, traceback.format_exc())
+ # except Exception as e:
+ # print('Kunde inte skriva error print till databasen.')
+ # print(e)
+ if "e" not in locals():
+ e = 'Unknown error'
+
doc = {
"_key": f"{now()}_{profile.container})",
"number": nr,
@@ -58,9 +70,9 @@ def write_error(nr, e="", traceback="", soup="", user="", url="", url_name="", p
"url": str(url),
"url_name": url_name,
"soup": str(soup),
- "traceback": str(traceback),
+ "traceback": str(traceback).split('\n'),
}
-
+
try:
db.insert_document(
"errors",
@@ -69,14 +81,17 @@ def write_error(nr, e="", traceback="", soup="", user="", url="", url_name="", p
silent=True,
)
except Exception as e:
- _print(profile.container, user, e)
+ _print(profile, user, e)
def now():
""" Returns current date and time as string"""
return datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
-
-def _print(container, user, text, end='\n', silent=False):
+def nowstamp():
+ """ Returns current date and time as timestamp"""
+ return int(datetime.now().timestamp())
+
+def _print(profile, user, text, end='\n', silent=False, sleeptime=0):
""" Write a "print" to the database (and prints in in the terminal)
Args:
@@ -86,15 +101,69 @@ def _print(container, user, text, end='\n', silent=False):
end (str, optional): The end value for print. Defaults to '\n'.
silent (bool, optional): If a print should be done in the terminal. Defaults to False.
"""
-
+
if silent == False:
print(text, end=end)
- if user != '':
- user = f"{user} - "
+
+ if profile.write == False:
+ return None
+
+ if profile.container[:4] == 'leak' and len(profile.container) < 7:
+ _key = f'{profile.container}_{now()[2:10]}'
+ elif profile.container[:7] == 'lookups':
+ _key = f'{profile.container}_{now()[2:10]}'
+ else:
+ _key = profile.container
+
+ try:
+ if isinstance(text, list):
+ to_print = {user.username: text}
+ else:
+ to_print = f"{user.username} - {text.strip()}"
+
+ except:
+ if isinstance(text, list):
+ to_print = {user: text}
+ else:
+ to_print = f"{text.strip()}"
+
+
db.insert_document(
"prints",
- {'_key': container, 'print':{now(): f"{user}{text.strip()}"}},
+ {'_key': _key, 'print':{now(): to_print}},
overwrite_mode="update",
silent=True,
- merge=True
- )
\ No newline at end of file
+ )
+ sleep(sleeptime)
+
+
+def check_profile_status(profile, user):
+ if profile.browser._cursor == -1: # Om ingen sida har öppnats än.
+ profile.open(url_bas)
+ if any(
+ [
+ "It looks like you were misusing this feature by going too fast."
+ in profile.viewing().text,
+ "Access Denied" in profile.viewing().text,
+ "Your Account Has Been Disabled" in profile.viewing().text
+ ]
+ ):
+ _print(profile, user, f"{profile.name} blocked\n".upper(), sleeptime=1)
+ _print(profile, user, profile.viewing().text, sleeptime=1)
+ profile.blocked = True # Nu tar jag bort dem, kan göras på annat sätt kanske?
+
+ elif "accept all" in profile.viewing().text.lower():
+ profile.accept_cookies()
+ sleep_(3)
+ profile.open(user.url_photos)
+ elif (
+ profile.viewing().find("title").text.strip() == "Log in to Facebook | Facebook"
+ ):
+ sleep_(5)
+ profile.login()
+ sleep_(5)
+ profile.open(user.url_photos)
+
+ return profile
+
+from arangodb import db
\ No newline at end of file
diff --git a/facebook/images.py b/facebook/images.py
new file mode 100644
index 0000000..352f27b
--- /dev/null
+++ b/facebook/images.py
@@ -0,0 +1,70 @@
+import requests
+import os
+from datetime import date, datetime, timedelta
+from time import sleep
+
+from arangodb import db
+
+
+def download_image(url, user, id):
+
+ # Kolla så användarmappen finns
+ if not os.path.isdir(f'../profile_pictures/{user}'):
+ os.mkdir(f'../profile_pictures/{user}')
+
+ # Ladda ner bilden
+ r = requests.get(url)
+ if r.text == 'URL signature expired':
+ print('För gammal länk.')
+ exit()
+ elif r.status_code == 403:
+ exit()
+ img_data = r.content
+ with open(f'../profile_pictures/{user}/{id}.jpg', 'wb') as handler:
+ handler.write(img_data)
+
+
+def get_pictures(day):
+ cursor = db.aql.execute(
+ """
+ for doc in members
+ filter doc.fetched == @date
+ filter has(doc, "checked_pictures")
+ filter not has(doc, "pictures_downloaded")
+ return {'member': doc._key, 'pictures':doc.checked_pictures}
+ """,
+ bind_vars={'date': day}
+ )
+
+ for doc in cursor:
+ pictures = []
+ for picture in doc['pictures']:
+ pictures.append(picture[picture.find('fbid=')+5:])
+
+
+ cursor = db.aql.execute(
+ """
+ for doc in pictures
+ filter doc._key in @list
+ limit 10
+ return {'_key': doc._key, 'user':doc.user, 'url': doc.src}
+ """,
+ bind_vars={"list": pictures},
+ )
+
+ for picture in cursor:
+ download_image(picture['url'], picture['user'], picture['_key'])
+ print(picture['_key'])
+ sleep(2)
+
+ db.update_document({'_id': 'members/' + str(doc['member']), 'pictures_downloaded': True}, silent=True, check_rev=False)
+
+def old_pics():
+ if not os.path.isdir(f'../profile_pictures'):
+ os.mkdir(f'../profile_pictures')
+ start = date.today()
+ for i in range(1,60):
+ d = start - timedelta(days=i)
+ get_pictures(d.strftime('%Y%m%d'))
+
+
diff --git a/facebook/images_pi.py b/facebook/images_pi.py
new file mode 100644
index 0000000..660773d
--- /dev/null
+++ b/facebook/images_pi.py
@@ -0,0 +1,169 @@
+import os
+from datetime import date, datetime, timedelta
+from getpass import getpass
+from time import sleep
+import random
+
+import requests
+import urllib3
+
+urllib3.disable_warnings()
+from arango import ArangoClient
+
+
+def download_image(url, user, id):
+
+ # Ladda ner bilden
+ while True:
+ try:
+ server = servers_mullvad[random.randint(0, len(servers_mullvad)-1)]
+ proxies = {
+ "https": "socks5://'8155249667566524'@{}".format(server),
+ "http": "socks5://'8155249667566524'@{}".format(server),
+ }
+ r = requests.get(url, proxies=proxies)
+ break
+ except requests.exceptions.ConnectionError:
+ sleep(300)
+
+ if r.text == "URL signature expired":
+ print("För gammal länk.")
+ exit()
+ elif r.status_code == 403:
+ exit()
+
+ image_name = f"/ssd/profile_pictures/{user}/{id}.jpg"
+ img_data = r.content
+ with open(image_name, "wb") as handler:
+ handler.write(img_data)
+
+ #nc_path = f"https://nc.lasseedfast.se/remote.php/dav/files/Lasse/profile_pictures/{user}/{id}.jpg"
+
+ # headers = {"Content-type": "image/jpeg", "Slug": "heart"}
+ # while True:
+ # try:
+ # r = requests.put(
+ # nc_path, data=open(image_name, "rb"), headers=headers, auth=auth, verify=False
+ # )
+ # break
+
+ # except:
+ # print('Kunde inte ladda upp', nc_path)
+ # sleep(5)
+
+ print(f"{user}\t{id}\t{r.status_code}")
+
+
+def get_pictures(day):
+ cursor = db.aql.execute(
+ """
+ for doc in members
+ filter doc.fetched == @date
+ filter has(doc, "checked_pictures")
+ filter not has(doc, "pictures_downloaded")
+ return {'member': doc._key, 'pictures':doc.checked_pictures}
+ """,
+ bind_vars={"date": str(day)},
+ )
+
+ # Skapa en lista med bilder att gå igenom.
+ images = []
+ for doc in cursor:
+ images.append(doc)
+ for doc in images:
+ user = doc["member"]
+
+ # # Skapa mapp för användarens bilder på NC...
+ # nc_path = f"https://nc.lasseedfast.se/remote.php/dav/files/Lasse/profile_pictures/{user}"
+ # while True:
+ # try:
+ # requests.request("MKCOL", nc_path, verify=False, auth=auth)
+ # break
+ # except:
+ # print('Kunde inte skapa', nc_path)
+ # sleep(5)
+
+ # ...och på datorn (för backup)
+ if not os.path.isdir(f"/ssd/profile_pictures/{user}"):
+ os.mkdir(f"/ssd/profile_pictures/{user}")
+
+ pictures = []
+ for picture in doc["pictures"]:
+ pictures.append(picture[picture.find("fbid=") + 5 :])
+
+ cursor = db.aql.execute(
+ """
+ for doc in pictures
+ filter doc._key in @list
+ limit 10
+ return {'_key': doc._key, 'user':doc.user, 'url': doc.src}
+ """,
+ bind_vars={"list": pictures},
+ )
+
+ for picture in cursor:
+ while True:
+ download_image(picture["url"], picture["user"], picture["_key"])
+ sleep(1)
+ break
+
+ db.update_document(
+ {"_id": "members/" + str(doc["member"]), "pictures_downloaded": True},
+ silent=True,
+ check_rev=False,
+ )
+
+
+# def old_pics():
+# if not os.path.isdir(f'profile_pictures'):
+# os.mkdir(f'profile_pictures')
+# start = date.today()
+# for i in range(1,60):
+# d = start - timedelta(days=i)
+# get_pictures(d.strftime('%Y%m%d'))
+
+
+if __name__ == '__main__':
+ # Info för arangodb
+ user_arango = "Lasse"
+ db_arango = "facebook"
+ host_arango = "http://192.168.0.4:8529"
+
+
+ # Starta koppling till arangodb
+ # Avkryptera lösen till arango
+ pwd = getpass(f"Arangolösenord för {user_arango}: ")
+
+
+ db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)
+ auth = ("Lasse", "affix-sip-jejune-epigraph-ENTROPY-stupefy1")
+
+ servers_mullvad = [
+ "se15-wg.socks5.mullvad.net:1080",
+ "se17-wg.socks5.mullvad.net:1080",
+ "se18-wg.socks5.mullvad.net:1080",
+ "se19-wg.socks5.mullvad.net:1080",
+ "se21-wg.socks5.mullvad.net:1080",
+ "se22-wg.socks5.mullvad.net:1080",
+ "se23-wg.socks5.mullvad.net:1080",
+ "se3-wg.socks5.mullvad.net:1080",
+ "se5-wg.socks5.mullvad.net:1080",
+ "se9-wg.socks5.mullvad.net:1080",
+ "se10-wg.socks5.mullvad.net:1080",
+ "se2-wg.socks5.mullvad.net:1080",
+ "se6-wg.socks5.mullvad.net:1080",
+ "se7-wg.socks5.mullvad.net:1080",
+ "se8-wg.socks5.mullvad.net:1080",
+ "se13-wg.socks5.mullvad.net:1080",
+ "se14-wg.socks5.mullvad.net:1080",
+ "se26-wg.socks5.mullvad.net:1080",
+ "se27-wg.socks5.mullvad.net:1080",
+ "se28-wg.socks5.mullvad.net:1080",
+ ]
+
+ while True:
+ today = date.today().strftime('%Y%m%d')
+ get_pictures(today)
+ yesterday = date.today() - timedelta(days=1)
+ get_pictures(yesterday.strftime('%Y%m%d'))
+ sleep(300)
diff --git a/facebook/modemtest.py b/facebook/modemtest.py
new file mode 100644
index 0000000..a427103
--- /dev/null
+++ b/facebook/modemtest.py
@@ -0,0 +1,45 @@
+import subprocess
+import requests
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from time import sleep
+
+while True:
+ wlan = subprocess.Popen(['iwgetid'], stdout=subprocess.PIPE)
+ wlan =wlan.communicate()[0].decode()
+ if '4G-UFI-5671' in wlan:
+ print('Sucess')
+ break
+ else:
+ sleep(20)
+
+print('Nuvarande ip:', requests.get('https://api.ipify.org').text)
+
+# Set up selenium browser
+options = Options()
+options.headless = True
+browser = webdriver.Chrome(options=options)
+
+# Login to modem
+browser.get('http://192.168.100.1/cellweb/login.asp')
+sleep(3)
+username = browser.find_element_by_id("user_name")
+password = browser.find_element_by_id("user_password")
+username.send_keys("admin")
+password.send_keys("1340asde")
+
+# Go to reboot and accept
+browser.find_element_by_xpath("/html/body/section/form/button").click() # Login
+sleep(1)
+browser.find_element_by_xpath("/html/body/section/div[2]/div[6]/a").click() # More
+sleep(1)
+browser.find_element_by_xpath("/html/body/section[2]/div/div[2]/div/a").click() # Reboot
+sleep(1)
+browser.find_element_by_xpath("/html/body/div[4]/div/div/div[2]/div[2]").click() # Accept
+sleep(1)
+browser.switch_to_alert().accept() # Accept again (alert)
+browser.close()
+sleep(120)
+
+
+print('Ny ip:', requests.get('https://api.ipify.org').text)
diff --git a/facebook/profile_create.py b/facebook/profile_create.py
index 484c255..571c0df 100644
--- a/facebook/profile_create.py
+++ b/facebook/profile_create.py
@@ -1,16 +1,21 @@
import random
import subprocess
from getopt import getopt
+from os import chdir
+from os.path import abspath, dirname
from sys import argv
from time import sleep
-from os import chdir
-from os.path import dirname
-from arangodb import db, get_profile, remove_profile
+# Gör fb-scraper till arbetsmapp
+chdir(dirname(dirname(abspath(__file__))))
+
+from arangodb import arango_connect, used_servers
+from config import *
from helpers import now
-def profile_generator(db, n):
+
+def profile_generator(db, n=1, bulk=False):
cursor = db.aql.execute(
"""
@@ -33,29 +38,36 @@ def profile_generator(db, n):
with open("data/passwords.txt") as f:
words = [word for word in f.readlines()]
+ servers_used = used_servers()
+
with open("data/servers.txt") as f:
servers = []
- for line in f.readlines():
- if "@" in line:
- line = line.strip()
- city = line[: line.find("@")].strip()
-
- if "WireGuard" in line and line.strip()[:2] in [
- "gb",
- "us",
- ]: # "au", "ca" #För senare när det behövs
- line = line.strip()
- country_short = line[:2]
- server = line[: line.find("-")]
- city_short = city[city.find("(") + 1 : city.find(")")]
- server_name = [country_short, city_short, server + "-wireguard"]
- servers.append(
- {
- "server_city": city,
- "server": server + "-wg.socks5.mullvad.net:1080",
- "server_connect": server_name,
- }
- )
+ while len(servers) < n:
+ for line in f.readlines():
+ if "@" in line:
+ line = line.strip()
+ city = line[: line.find("@")].strip()
+
+ if "WireGuard" in line and line.strip()[:2] in [
+ "gb",
+ "us",
+ ]: # "au", "ca" #För senare när det behövs
+ line = line.strip()
+ country_short = line[:2]
+ server = line[: line.find("-")]
+
+ # Kolla så att servern inte redan används av profil i databasen
+ # eller finns i listan som skapas nu.
+ if server not in servers_used and server not in servers:
+ city_short = city[city.find("(") + 1 : city.find(")")]
+ server_name = [country_short, city_short, server + "-wireguard"]
+ servers.append(
+ {
+ "server_city": city,
+ "server": server + "-wg.socks5.mullvad.net:1080",
+ "server_connect": server_name,
+ }
+ )
count = 0
for i in range(0, n - 1):
@@ -79,6 +91,8 @@ def profile_generator(db, n):
server = server_info["server"].strip()
birthday = f"{year}-{random.randrange(1, 13)}-{random.randrange(1, 30)}"
_key = server[: server.find("-")]
+
+
doc = {
"_id": "profiles/" + _key,
"_key": _key,
@@ -92,17 +106,23 @@ def profile_generator(db, n):
"in_use": False,
}
# Skriv till databasen (skriver inte profiler med servarar som redan används)
- try:
- db.insert_document("profiles", doc)
- count += 1
- except:
- pass
+
+ if bulk == True:
+ try:
+ db.insert_document("profiles", doc)
+ count += 1
+ except:
+ pass
+
+ print(f"Skrev {count} profiler till databasen. ")
- print(f"Skrev {count} profiler till databasen. ")
+ else:
+ return doc
def mullvad(server):
""" Anslut till Mullvad-server. """
+ sleep(2)
subprocess.run(
[
"mullvad",
@@ -118,6 +138,9 @@ def mullvad(server):
connect_to_mullvad.wait()
sleep(3)
+def close_browser():
+ subprocess.run(["osascript", "-e", 'quit app "Brave Browser"'])
+ subprocess.run(["osascript", "-e", 'quit app "Opera"'])
def create_profile():
""" Supports during the creation of a profile """
@@ -125,9 +148,10 @@ def create_profile():
while True:
mullvad(arango_server)
-
+ sleep(2)
# Hämta profil
- profile = get_profile(created=False)
+ db = arango_connect(pwd)
+ profile = profile_generator(db)
# Asnlut till profilens VPN-server
mullvad(profile["server_connect"])
@@ -142,7 +166,6 @@ def create_profile():
print(profile["birthday"])
print(profile["name"])
- print()
print()
print(profile["name"])
print(profile["email"])
@@ -153,38 +176,45 @@ def create_profile():
user_input = user_input.lower()
if user_input in ["done", "d", ""]:
- subprocess.run(["osascript", "-e", 'quit app "Brave Browser"'])
+ close_browser()
sleep(1)
profile["created"] = True
mullvad(arango_server)
sleep(3)
- db.update_document(profile)
+ db = arango_connect(pwd)
+ db.update_document('profiles', profile)
elif user_input in ["delete", "d"]:
- subprocess.run(["osascript", "-e", 'quit app "Brave Browser"'])
- sleep(1)
+ close_browser()
mullvad(arango_server)
- blocked = remove_profile(profile)
+ db = arango_connect(pwd)
+ blocked = db.collection("profiles").delete(profile['_key'])
blocked['_id'] = 'block_created/' + now()
blocked['key'] = now()
+ sleep(2)
+ db = arango_connect(pwd)
db.insert_document("block_created", blocked)
elif user_input in ["quit", "q"]:
- subprocess.run(["osascript", "-e", 'quit app "Brave Browser"'])
- sleep(1)
+ close_browser()
+ close_browser()
mullvad(arango_server)
- sleep(3)
exit()
else:
+ sleep(3)
+ db = arango_connect(pwd)
continue
if __name__ == "__main__":
print(__file__)
- # Säkerställ att arbetsmappen är samma som den där scriptet ligger
- chdir(dirname(__file__))
- subprocess.run(['cd', '..'])
+ # Det här ska köras lokalt så löseordet finns i fil
+ with open('password_arango.txt') as f:
+ pwd = f.readline()
+
+ db = arango_connect(pwd)
+
argv = argv[1:]
opts, args = getopt(argv, "cg:", ["create", "generate"])
@@ -193,3 +223,6 @@ if __name__ == "__main__":
profile_generator(db, a)
if o in ['-c', '--create']:
create_profile()
+
+
+[{'domain': '.facebook.com', 'httpOnly': False, 'name': 'x-referer', 'path': '/', 'sameSite': 'None', 'secure': True, 'value': 'eyJyIjoiL2NvbmZpcm1lbWFpbC5waHA%2Fc29mdD1oamsiLCJoIjoiL2NvbmZpcm1lbWFpbC5waHA%2Fc29mdD1oamsiLCJzIjoibSJ9'}, {'domain': '.facebook.com', 'expiry': 1649728634, 'httpOnly': True, 'name': 'xs', 'path': '/', 'secure': True, 'value': '2%3AZrCj3xPTmzApJw%3A2%3A1618192633%3A-1%3A-1'}, {'domain': '.facebook.com', 'expiry': 1649728634, 'httpOnly': False, 'name': 'c_user', 'path': '/', 'secure': True, 'value': '100066462633263'}, {'domain': '.facebook.com', 'expiry': 1618797592, 'httpOnly': False, 'name': 'wd', 'path': '/', 'sameSite': 'None', 'secure': True, 'value': '994x534'}, {'domain': '.facebook.com', 'httpOnly': False, 'name': 'm_pixel_ratio', 'path': '/', 'secure': True, 'value': '1.25'}, {'domain': '.facebook.com', 'expiry': 1625968625, 'httpOnly': True, 'name': 'fr', 'path': '/', 'secure': True, 'value': '16Qhs4a4NEktNwlhZ.AWXIpZOVbupyu5pAidanfvTaWIc.Bgc6jD.C2.AAA.0.0.Bgc6jy.AWXCdEVJ7k4'}, {'domain': '.facebook.com', 'expiry': 1681264643, 'httpOnly': True, 'name': 'sb', 'path': '/', 'secure': True, 'value': 'w6hzYAXTtE1avdx0LoFIrHox'}, {'domain': '.facebook.com', 'expiry': 1681264587, 'httpOnly': True, 'name': 'datr', 'path': '/', 'secure': True, 'value': 'w6hzYONZsuS635di6pHBZV7D'}]
\ No newline at end of file
diff --git a/facebook/scrapers.py b/facebook/scrapers.py
index 185d0b2..0f46bfc 100644
--- a/facebook/scrapers.py
+++ b/facebook/scrapers.py
@@ -1,41 +1,34 @@
import re
import traceback
+import requests
-from arangodb import db
+from arangodb import db, check_for_picture
from classes import Friend, Picture, Reaction
from config import *
-from helpers import sleep_, update_cookie, write_error, _print
+from helpers import sleep_, update_cookie, write_error, _print, check_profile_status
-def profile_picture_reactions(
- profile, user, all_pictures, first_user=False, mode="all"
-):
+def profile_picture_reactions(profile, user, first_user=False, mode="all"):
+ # try:
# Fixa url:er osv
+
if user.username.isnumeric():
user.url = url_bas + "/profile.php?id=" + str(user.username)
user.url_photos = user.url + "&v=photos"
+ user.id = user.username
+
else:
user.username = user.username.replace("/", "")
user.url = url_bas + "/" + user.username
user.url_photos = user.url + "/photos"
# Gå till sidan för profilbilder
- profile.browser.open(user.url_photos)
-
+ profile.open(user.url_photos)
+ # print(profile.viewing())
sleep_(4)
- if (
- """You can't use Facebook because your account, or activity on it, doesn't follow our Community Standards."""
- in profile.viewing().text
- ):
- _print(profile.container, user.username, f"{profile.name} blocked\n".upper())
- profile.blocked = True
- return "blocked"
-
- elif "accept all" in profile.viewing().text.lower():
- profile.accept_cookies()
- profile.browser.open(user.url_photos)
+ profile = check_profile_status(profile, user)
user.name = user.username # Om inte namnet hittas senare
try:
@@ -47,23 +40,27 @@ def profile_picture_reactions(
except Exception as e:
write_error(
6,
+ profile,
e=e,
traceback=traceback.format_exc(),
- profile=profile.container,
soup=profile.viewing(),
user=user,
url=user.url_photos,
)
if first_user == True:
- _print(profile.container, user.username, profile.viewing().prettify())
+ _print(profile, user, profile.viewing().prettify())
exit()
- _print(profile.container, user.username, f"Hämtar reaktioner på profilbilder för {user.name} ({user.username})")
+ _print(
+ profile,
+ user,
+ f"Hämtar reaktioner på profilbilder för {user.name} ({user.username})",
+ )
# Hitta länk till olika saker hos användarem, inkl facebook-id
for a in profile.viewing().find_all("a", href=True):
if "Profile pictures" in a.text:
- user.url_album = url_bas + a["href"] # Länk till album för profilbulder
+ user.url_album = url_bas + a["href"] # Länk till album för profilbilder
if "profile_id" in a["href"]:
l = a["href"]
try:
@@ -75,10 +72,13 @@ def profile_picture_reactions(
user.url_likes = url_bas + a["href"]
if "About" in a.text:
user.url_about = url_bas + a["href"]
+ user.id = user.url_about[user.url_about.find('%')+3: user.url_about.rfind('%')]
if "Timeline" in a.text:
user.url_timeline = url_bas + a["href"]
if "Cover photos" in a.text:
user.url_coverphotos = url_bas + a["href"]
+ if a.text == "Friends":
+ user.url_friends = url_bas + a["href"]
# Om det inte finns något profilalbum
# Testa ta bort mellanrum och små bokstäver
@@ -90,78 +90,84 @@ def profile_picture_reactions(
user.add_to_db()
# Gå till profilbilden (den första som kommer upp när man går till profilen)
- if not hasattr(user, "url_album"): # Om profilen inte har profilalbum
- write_error(9, soup=profile.viewing(), user=user, profile=profile.container)
- if user.url_other_picture != "":
- # Använd eventuell extrabild och ta bort den från användaren
- url_pics = [user.url_other_picture]
- user.url_other_picture = ""
+ # Om profilen inte har profilalbum
+ if not hasattr(user, "url_album"):
+ write_error(9, profile, soup=profile.viewing(), user=user)
+ if user.url_other_pictures != []:
+ # Använd eventuella extrabilder och ta bort den från användaren
+ url_pics = user.url_other_pictures
+ user.url_other_pictures = []
else:
# Spara ner profilen till databasen och avsluta sökningen på användaren
user.url_album = False
if first_user == False:
user.checked()
user.add_to_db()
- _print(profile.container, user.username, "Hittar inget album för profilbilder.")
- write_error(#fel7
+ _print(profile, user, "Hittar inget album för profilbilder.")
+ write_error( # fel7
7,
+ profile,
soup=profile.viewing(),
- profile=profile.container,
user=user,
url=user.url_album,
url_name="user.url_album",
)
- return None
+ return profile
# ATT GÖRA Här kan andra bilder väljas istället
- else: # Normalfallet där användaren har profilbildsalbum
- profile.browser.open(user.url_album)
+ # Normalfallet där användaren har profilbildsalbum
+ else:
+ profile.open(user.url_album)
# Samla alla profilbilder i en lista
- url_pics = []
+ url_pics = user.url_other_pictures
pics = profile.viewing().find("div", {"id": "thumbnail_area"})
for i in pics.find_all("a"):
a = i["href"]
url_pics.append(a[: a.find("&id")])
- if user.url_other_picture != "":
- # Lägg till eventuell extrabild och ta bort den från användaren
- url_pics.append(user.url_other_picture)
- user.url_other_picture = ""
try:
user.profile_pictures = len(url_pics)
+ user.pictures = url_pics
except:
+ _print(profile, user, "Hittade inga profilbilder".upper())
user.profile_pictures = 0
+ user.pictures = url_pics
user.checked()
user.add_to_db()
- return
+ return profile
# Lägg till profilen till arrango.
user.add_to_db()
- # Välj vilja bilder som ska kollas.
- if first_user == False:
- if mode == "single":
- url_pics = url_pics[:1]
- elif mode == "few" and len(url_pics) > 1:
- url_pics = url_pics[:1] + url_pics[-1:]
-
# Gå igenom valda bilder.
for pic in url_pics:
+ if check_for_picture(pic[pic.find("fbid=") + 5 :]):
+ continue
+
# Skriv ut vilken bild som behandlas.
- _print(profile.container, user.username,
+ _print(
+ profile,
+ user,
f"Bild {url_pics.index(pic) + 1} av {user.profile_pictures}",
end="\r",
)
check_picture(url_bas + pic, user, profile)
-
+ user.checked_pictures.append(url_bas + pic)
+
# Välj vilja bilder som ska kollas.
if first_user == False:
if mode == "single" and user.reactions > 30:
break
- elif all([mode == "few", user.reactions > 50, pic != url_pics[-1]]):
+ elif all([any[mode == "few", mode == "solo"], user.reactions > 80, pic != url_pics[-1]]):
# Kolla den sista bilder
check_picture(url_bas + url_pics[-1], user, profile)
+ user.checked_pictures.append(url_bas + pic)
break
user.checked()
+ return profile
+
+ # except Exception as e:
+ # _print(None, str(e))
+ # return profile
def check_picture(url_picture, user, profile):
@@ -179,12 +185,12 @@ def check_picture(url_picture, user, profile):
sleep_(5)
try:
- profile.browser.open(picture.url)
+ profile.open(picture.url)
except Exception as e: # Fel3
write_error(
3,
+ profile,
e=e,
- profile=profile.container,
soup=profile.viewing(),
user=user,
url=picture.url,
@@ -192,7 +198,7 @@ def check_picture(url_picture, user, profile):
traceback=traceback.format_exc(),
)
- update_cookie(profile.browser.session.cookies, profile.name)
+ update_cookie(profile.browser.session.cookies, profile)
# Hitta info om bilden
try:
@@ -202,16 +208,23 @@ def check_picture(url_picture, user, profile):
8,
e=e,
soup=profile.viewing(),
- profile=profile.container,
+ profile=profile,
url=picture.url,
url_name="picture url",
user=user,
traceback=traceback.format_exc(),
)
- # TODO #3 lägg till fler bilder som kan gås igenom om det är få profilbilder.
- # Hämta länkar för bilden att anvrända sen
- # _print(profile.container, user.username, profile.viewing().prettify())
+ try:
+ for img in profile.viewing().find_all('img'):
+ if 'https://scontent' in img['src']:
+ picture.src = img['src']
+
+ except Exception as e:
+ pass
+
+ # Hämta länkar för bilden att använda sen
+ # _print(profile, user, profile.viewing().prettify())
for a in profile.viewing().find_all("a", href=True):
if all(
[
@@ -240,9 +253,9 @@ def check_picture(url_picture, user, profile):
# Hämta reaktioner för bilden
sleep_(3)
- profile.browser.open(url_reactions)
+ profile.open(url_reactions)
- update_cookie(profile.browser.session.cookies, profile.name)
+ update_cookie(profile.browser.session.cookies, profile)
try:
for a in profile.viewing().find_all("a", {"class": "z ba"}, href=True):
@@ -255,15 +268,15 @@ def check_picture(url_picture, user, profile):
except UnboundLocalError: # fel9
write_error(
9,
+ profile,
user=user,
- profile=profile.container,
soup=profile.viewing(),
traceback=traceback.format_exc(),
url=url_reactions,
url_name="url_reactions",
)
# Bilder med väldigt många likes går inte att visa så här?
- return None
+ return profile
# Addera bilden till arrango
picture.add_to_db()
@@ -274,9 +287,9 @@ def check_picture(url_picture, user, profile):
try:
sleep_(4)
- profile.browser.open(url_limit)
+ profile.open(url_limit)
url_limit = ""
- update_cookie(profile.browser.session.cookies, profile.name)
+ update_cookie(profile.browser.session.cookies, profile)
# Gå igenom alla som reagerat och för in i arango
for li in profile.viewing().find_all("li"):
@@ -288,9 +301,19 @@ def check_picture(url_picture, user, profile):
friend.name = friend_html.text
friend.url = friend_html["href"]
if "profile.php" in friend.url:
- friend.username = friend.url[friend.url.find("id=") + 3 :]
+ if "&paipv" in friend.url:
+ friend.username = friend.url[
+ friend.url.find("=") + 1 : friend.url.find("&")
+ ]
+ else:
+ friend.username = friend.url[friend.url.find("id=") + 3 :]
else:
- friend.username = friend.url[friend.url.find("/") + 1 :]
+ if "?" in friend.url:
+ friend.username = friend.url[
+ friend.url.find("/") + 1 : friend.url.find("?")
+ ]
+ else:
+ friend.username = friend.url[friend.url.find("/") + 1 :]
reaction = Reaction(user.username, friend.username, picture.id)
for type in ["Love", "Wow", "Like", "Care", "Sad", "Angry", "Haha"]:
@@ -298,19 +321,22 @@ def check_picture(url_picture, user, profile):
reaction.type = type
picture.reactions.append(reaction.get_dict())
# Lägg till vännens profil till arrango
- friend.add_to_db()
-
+ try:
+ friend.add_to_db()
+ except:
+ _print(profile, user, f"Kunde inte lägga till vän {friend.url}")
+
except AttributeError as e: # Fel1
write_error(
1,
+ profile,
e=e,
soup=str(li),
user=user,
- profile=profile.container,
traceback=traceback.format_exc(),
)
pass
-
+
# Lägg till reaktioner till databasen
db.collection("picture_reactions").insert_many(
picture.reactions, silent=True, overwrite=True
@@ -318,18 +344,19 @@ def check_picture(url_picture, user, profile):
db.collection("picture_reactions").insert_many(
picture.reactions, silent=True, overwrite=True
)
-
+
# Uppdatera antalet reaktioner användaren fått
user.reactions += len(picture.reactions)
except Exception as e: # Fel2
write_error(
2,
+ profile,
e=e,
soup=profile.viewing(),
- profile=profile.container,
user=user,
url=url_limit,
url_name="url_limit",
traceback=traceback.format_exc(),
)
- pass
\ No newline at end of file
+ pass
+
diff --git a/facebook/socks5free.csv b/facebook/socks5free.csv
new file mode 100644
index 0000000..30c2e18
--- /dev/null
+++ b/facebook/socks5free.csv
@@ -0,0 +1,557 @@
+DE;91.198.137.31:3552;FAIL;
+FR;54.36.4.70:61432;188.165.211.29;0:00:00.601284
+DE;138.68.82.88:1080;138.68.82.88;0:00:05.222863
+DE;172.104.142.154:35819;172.104.142.154;0:00:02.972221
+MD;185.14.31.113:443;FAIL;
+NL;146.185.132.87:31284;146.185.132.87;0:00:00.541678
+UA;46.151.197.254:8080;46.151.197.254;0:00:02.210311
+FI;135.181.184.170:54048;135.181.184.170;0:00:00.720659
+FR;151.106.34.139:1080;FAIL;
+NL;88.202.177.242:1090;FAIL;
+UA;95.67.99.99:33871;FAIL;
+DE;138.201.5.46:1080;138.201.5.46;0:00:07.487491
+DE;159.69.106.103:1080;159.69.106.103;0:00:00.630095
+NL;51.15.78.50:1080;51.15.78.50;0:00:00.564266
+NL;88.202.177.242:1080;FAIL;
+SG;113.77.85.215:1081;149.129.48.241;0:00:03.688375
+RU;95.107.37.109:1105;FAIL;
+KZ;109.229.161.151:1225;FAIL;
+RU;84.22.137.26:9025;84.22.137.26;0:00:09.468929
+US;149.28.126.83:1081;149.28.126.83;0:00:01.023434
+AR;181.3.58.168:1080;FAIL;
+US;67.227.193.162:34496;67.227.193.162;0:00:01.318698
+NL;146.185.132.87:44795;146.185.132.87;0:00:02.952634
+US;198.58.119.187:50398;69.164.194.35;0:00:01.449008
+AR;186.126.79.171:1080;FAIL;
+CA;192.252.211.197:14921;FAIL;
+CA;192.252.209.155:14455;FAIL;
+CZ;89.187.144.153:1080;89.187.144.153;0:00:01.096993
+US;209.141.53.246:1080;FAIL;
+US;192.111.137.37:18762;FAIL;
+CA;192.252.208.67:14287;FAIL;
+US;67.55.185.240:1888;FAIL;
+NL;142.93.137.235:54866;142.93.137.235;0:00:04.162599
+US;192.111.135.18:18301;FAIL;
+US;192.111.138.29:4145;FAIL;
+SG;45.77.36.30:24574;45.77.36.30;0:00:02.664875
+US;70.185.68.133:4145;FAIL;
+FR;51.68.134.242:25623;51.68.134.240;0:00:05.998615
+FR;193.70.45.126:32821;193.70.45.126;0:00:03.586748
+US;98.162.25.7:31653;FAIL;
+US;72.223.168.86:57481;FAIL;
+DE;213.136.89.190:18461;213.136.89.190;0:00:03.407266
+DE;101.53.158.48:9051;FAIL;
+PL;5.226.69.12:50477;5.226.69.12;0:00:08.327345
+US;98.162.25.29:31679;FAIL;
+IN;103.209.64.19:6667;FAIL;
+AR;186.126.42.65:1080;200.73.130.62;0:00:11.137412
+US;72.221.196.157:35904;FAIL;
+US;72.206.181.105:64935;FAIL;
+SG;113.77.86.73:1081;149.129.55.120;0:00:02.697133
+CA;192.252.214.20:15864;FAIL;
+RU;109.72.231.37:1080;109.72.231.37;0:00:01.095943
+TR;188.132.179.124:60088;188.132.179.124;0:00:01.228944
+AR;181.3.72.8:1080;200.73.132.176;0:00:17.562909
+GB;157.245.34.127:32215;157.245.34.127;0:00:06.500380
+RU;31.7.232.178:1080;31.7.232.178;0:00:08.192440
+US;72.223.168.73:57494;FAIL;
+AR;186.126.135.164:1080;200.73.130.62;0:00:14.713391
+IN;165.22.220.151:36362;165.22.220.151;0:00:05.533314
+US;192.111.137.35:4145;FAIL;
+BR;186.126.143.88:1080;FAIL;
+BR;181.3.51.12:1080;209.14.2.12;0:00:08.158021
+US;104.238.215.49:1080;104.238.215.49;0:00:04.300450
+AR;186.126.163.43:1080;FAIL;
+BR;181.6.94.90:1080;FAIL;
+AR;181.3.23.13:1080;FAIL;
+VN;113.160.188.21:1080;113.160.188.21;0:00:05.010119
+FI;135.181.184.170:22497;135.181.184.170;0:00:09.929478
+FR;51.68.134.247:30204;FAIL;
+AR;181.101.2.92:1080;200.73.132.106;0:00:03.141000
+BR;181.101.26.136:1080;FAIL;
+BR;181.3.71.184:1080;191.252.103.251;0:00:03.371414
+AR;181.102.21.228:1080;FAIL;
+AR;181.3.37.59:1080;FAIL;
+AR;186.126.177.123:1080;FAIL;
+AR;186.126.151.29:1080;45.235.98.221;0:00:05.725761
+AR;181.3.61.217:1080;FAIL;
+AR;181.5.222.133:1080;FAIL;
+FR;51.68.134.241:30204;FAIL;
+HK;1.65.196.134:1080;1.65.196.134;0:00:04.107338
+BR;181.3.8.101:1080;54.232.66.92;0:00:25.527846
+UA;91.229.123.191:1080;94.153.23.177;0:00:07.271681
+AR;181.3.62.188:1080;FAIL;
+KR;222.99.47.68:8888;FAIL;
+KR;119.28.73.113:22225;158.247.225.109;0:00:02.975846
+AR;181.102.5.177:1080;45.235.99.87;0:00:04.846713
+AR;181.101.16.232:1080;FAIL;
+AR;181.101.12.108:1080;FAIL;
+AR;181.101.38.248:1080;45.235.99.87;0:00:12.370835
+AR;181.3.59.102:1080;FAIL;
+FR;195.154.178.247:20152;FAIL;
+DE;46.101.218.6:24040;46.101.218.6;0:00:02.524995
+US;173.236.188.154:7595;173.236.184.102;0:00:07.522997
+AR;181.3.4.18:1080;200.69.236.22;0:00:03.333511
+CA;181.101.14.230:1080;FAIL;
+DK;142.93.245.247:30588;FAIL;
+FR;54.36.4.69:61432;188.165.211.29;0:00:00.679880
+AR;186.152.120.155:1080;45.235.99.88;0:00:13.682541
+IN;27.116.51.181:6667;FAIL;
+AR;181.7.201.154:1080;45.235.99.83;0:00:03.619538
+FR;51.68.134.245:25623;51.68.134.240;0:00:03.046891
+US;192.111.139.165:19402;FAIL;
+AR;186.126.140.70:1080;FAIL;
+US;184.178.172.5:15303;FAIL;
+AR;186.126.25.102:1080;200.73.134.139;0:00:18.534001
+US;181.3.66.118:1080;FAIL;
+BR;186.126.141.239:1080;177.67.82.171;0:00:20.168977
+DE;78.46.200.13:22039;78.46.200.13;0:00:03.381044
+BR;186.152.119.220:1080;FAIL;
+AR;186.152.33.185:10808;FAIL;
+US;181.102.84.53:1080;FAIL;
+AR;186.152.31.215:1080;FAIL;
+BR;186.152.194.140:1080;FAIL;
+US;173.236.184.154:22960;173.236.184.139;0:00:02.895083
+FR;137.74.153.106:1080;137.74.153.106;0:00:03.010125
+AR;186.126.32.22:1080;138.99.7.145;0:00:07.475672
+BR;181.101.11.43:1080;FAIL;
+US;72.210.252.134:46164;FAIL;
+BR;181.3.56.124:1080;FAIL;
+AR;181.101.47.84:1080;FAIL;
+CA;181.6.141.73:1080;FAIL;
+MD;178.175.139.202:57772;178.175.139.202;0:00:01.611892
+PH;210.16.73.82:1080;124.107.231.80;0:00:03.173570
+AR;186.126.44.155:1080;200.89.175.133;0:00:08.703594
+BR;181.101.60.197:1080;104.41.41.29;0:00:07.245720
+KR;125.135.221.94:54557;FAIL;
+US;186.126.62.200:1080;FAIL;
+GB;178.62.79.115:35580;178.62.79.115;0:00:05.262268
+FI;95.216.176.163:1089;95.216.176.163;0:00:09.142730
+CA;186.126.21.113:1080;FAIL;
+AR;181.3.38.147:1080;181.117.241.51;0:00:04.966959
+US;70.166.167.38:57728;FAIL;
+AR;181.3.78.111:1080;200.73.131.75;0:00:09.585425
+BR;181.5.244.219:1080;FAIL;
+FR;51.68.134.240:25623;51.68.134.240;0:00:08.593545
+US;181.102.16.72:1080;FAIL;
+FR;178.32.47.218:50939;178.32.47.218;0:00:06.439677
+US;173.236.189.175:22960;FAIL;
+AR;181.7.208.112:1080;FAIL;
+IN;103.241.227.110:6667;FAIL;
+US;147.135.116.172:53079;147.135.116.172;0:00:02.112520
+AR;186.126.64.146:1080;FAIL;
+CA;181.0.12.116:1080;FAIL;
+US;198.8.94.170:39074;FAIL;
+AR;181.3.76.4:1080;FAIL;
+AR;181.7.204.60:1080;FAIL;
+AR;181.3.28.148:1080;FAIL;
+BR;181.3.74.230:1080;45.162.231.55;0:00:13.378087
+US;113.73.72.183:1080;FAIL;
+US;141.98.134.2:1080;141.98.134.2;0:00:03.583016
+CA;192.111.130.5:17002;FAIL;
+RU;185.233.202.27:1080;185.233.202.27;0:00:11.702264
+DE;173.212.201.250:47492;173.212.201.250;0:00:07.449093
+SG;206.189.158.28:7905;206.189.158.28;0:00:08.228267
+US;173.236.190.7:7595;173.236.184.102;0:00:05.519787
+US;173.236.188.46:22960;173.236.184.139;0:00:05.490614
+US;173.236.185.99:22960;173.236.184.139;0:00:09.586001
+AR;186.126.73.156:1080;200.73.130.62;0:00:03.150311
+GB;157.245.34.127:61851;157.245.34.127;0:00:04.082666
+TW;60.169.205.61:1080;FAIL;
+BR;181.101.47.97:1080;191.233.232.45;0:00:03.439772
+FR;51.68.134.253:25623;51.68.134.240;0:00:08.526576
+AR;181.3.16.106:10808;FAIL;
+US;173.236.190.93:22960;FAIL;
+US;186.126.99.163:10808;FAIL;
+AR;186.152.130.181:1080;200.73.138.194;0:00:10.460878
+AR;186.152.15.200:1080;FAIL;
+AR;181.5.232.149:1080;FAIL;
+DE;165.22.17.195:5110;165.22.17.195;0:00:04.337353
+FR;51.68.134.244:25623;51.68.134.240;0:00:05.794034
+AR;186.126.80.182:1080;FAIL;
+SG;206.189.158.28:53176;206.189.158.28;0:00:04.394778
+AR;186.126.3.27:1080;FAIL;
+AR;186.126.17.42:1080;FAIL;
+BR;186.126.159.136:1080;FAIL;
+BR;186.126.70.165:1080;54.207.134.244;0:00:07.969362
+CL;181.5.217.57:1080;FAIL;
+US;66.42.224.229:41679;FAIL;
+BR;181.3.2.188:1080;FAIL;
+FR;51.68.134.252:25623;51.68.134.240;0:00:03.820479
+BR;181.83.226.81:1080;209.14.2.204;0:00:10.407002
+US;104.238.212.43:1081;104.238.212.43;0:00:03.010979
+BR;186.126.109.207:1080;FAIL;
+BR;181.3.39.114:1080;FAIL;
+FR;51.68.134.255:22308;51.68.134.240;0:00:05.837994
+US;184.178.172.18:15280;FAIL;
+FR;51.68.134.247:25623;51.68.134.240;0:00:05.294231
+AR;181.3.84.123:1080;45.235.99.87;0:00:07.781855
+AR;186.126.51.206:1080;FAIL;
+BR;181.83.228.198:1080;FAIL;
+AR;186.126.40.168:1080;FAIL;
+US;181.0.8.189:1080;FAIL;
+AR;181.101.35.11:1080;FAIL;
+US;104.238.111.218:57978;104.238.111.218;0:00:06.871360
+CA;181.3.20.113:10808;FAIL;
+FR;51.75.42.95:25623;51.68.134.240;0:00:04.044253
+US;173.236.187.212:22960;173.236.184.139;0:00:03.293691
+BR;181.3.65.241:1080;FAIL;
+US;173.236.186.231:22960;173.236.184.139;0:00:03.276001
+US;165.227.177.113:24586;165.227.177.113;0:00:05.401278
+CA;186.126.58.189:10808;51.222.141.137;0:00:04.245833
+DE;176.9.160.118:22836;FAIL;
+US;74.208.101.185:31200;FAIL;
+US;186.126.166.22:1080;FAIL;
+AR;181.101.33.157:1080;45.235.99.83;0:00:05.120106
+AR;186.126.110.76:1080;FAIL;
+US;186.126.170.254:1080;FAIL;
+FR;51.68.134.251:25623;51.68.134.240;0:00:06.095322
+BR;186.126.74.124:1080;FAIL;
+IN;43.224.10.32:6667;FAIL;
+US;95.217.132.133:3178;FAIL;
+US;157.230.154.211:28030;157.230.154.211;0:00:03.840172
+IN;140.238.250.54:1080;140.238.250.54;0:00:04.823383
+AR;181.102.134.167:1080;FAIL;
+AR;186.126.101.52:1080;FAIL;
+CO;181.129.7.202:6699;181.129.7.202;0:00:02.020779
+US;186.126.15.241:10808;FAIL;
+AR;181.101.8.41:1080;FAIL;
+AR;181.0.0.18:1080;FAIL;
+SG;181.3.58.52:1080;FAIL;
+AR;181.101.9.46:1080;FAIL;
+SG;129.226.196.49:41789;129.226.196.49;0:00:02.896387
+CA;192.111.129.145:16894;FAIL;
+AR;181.3.51.132:1080;FAIL;
+AR;181.3.10.74:1080;FAIL;
+BR;181.3.9.61:1080;FAIL;
+AR;181.3.49.78:1080;FAIL;
+GB;181.101.52.44:1080;FAIL;
+US;69.61.200.104:36181;FAIL;
+BR;186.126.177.239:1080;FAIL;
+BR;186.152.122.42:1080;FAIL;
+CL;186.126.71.210:1080;170.239.87.87;0:00:10.699452
+US;184.178.172.13:15311;FAIL;
+BD;103.85.232.146:1080;FAIL;
+US;161.35.137.49:28005;FAIL;
+AR;181.101.45.131:1080;FAIL;
+US;70.166.167.55:57745;FAIL;
+AR;181.3.57.187:1080;FAIL;
+NL;188.166.104.152:6683;FAIL;
+US;95.217.132.133:3038;FAIL;
+IN;103.241.227.98:6667;FAIL;
+AR;181.102.47.46:1080;FAIL;
+PL;5.226.69.12:41284;5.226.69.12;0:00:05.842418
+AR;186.126.139.224:10808;FAIL;
+AR;181.7.197.13:1080;FAIL;
+AR;186.152.16.246:1080;FAIL;
+US;113.73.72.177:1080;FAIL;
+US;72.221.164.34:60671;FAIL;
+BR;181.3.68.127:1080;FAIL;
+US;173.236.186.236:22960;173.236.184.139;0:00:03.567567
+AR;186.126.167.68:1080;FAIL;
+IN;103.240.168.138:6667;FAIL;
+US;104.248.0.141:17074;FAIL;
+AR;181.5.219.126:1080;FAIL;
+CA;186.152.115.63:1080;FAIL;
+US;132.148.129.108:34289;132.148.129.108;0:00:06.245162
+AR;186.126.138.242:1080;FAIL;
+AR;181.102.16.55:1080;FAIL;
+US;104.238.215.49:1081;104.238.212.43;0:00:07.598953
+US;147.135.116.172:26522;147.135.116.172;0:00:03.047146
+GB;178.62.79.49:51591;178.62.79.49;0:00:04.168867
+AR;181.3.39.27:1080;FAIL;
+BR;181.6.149.14:1080;201.76.56.248;0:00:10.817129
+IN;27.116.51.85:6667;FAIL;
+IN;103.216.82.22:6667;FAIL;
+SG;206.189.158.28:44880;206.189.158.28;0:00:10.378409
+SK;109.74.144.149:22743;109.74.144.149;0:00:07.030135
+FR;51.68.134.241:25623;51.68.134.240;0:00:08.225295
+AR;181.6.8.208:10808;200.73.132.2;0:00:14.850405
+AR;186.152.4.160:1080;FAIL;
+AR;181.3.46.25:1080;FAIL;
+US;208.102.51.6:58208;FAIL;
+AR;181.101.53.240:1080;200.73.132.115;0:00:09.802936
+IN;103.251.225.16:6667;FAIL;
+US;173.236.185.19:22960;FAIL;
+FR;51.68.134.250:25623;FAIL;
+US;50.62.35.16:41644;50.62.35.16;0:00:02.304961
+BR;186.126.129.193:1080;FAIL;
+US;166.62.85.224:13954;166.62.85.224;0:00:05.123121
+US;47.100.88.171:20900;FAIL;
+US;104.238.111.167:14416;FAIL;
+US;64.34.217.33:40741;FAIL;
+CA;192.252.215.5:16137;FAIL;
+US;173.236.184.139:22960;173.236.184.139;0:00:04.575732
+DE;46.101.218.6:39749;46.101.218.6;0:00:06.758081
+AR;181.101.2.18:1080;FAIL;
+US;66.228.36.18:61852;FAIL;
+DE;173.212.201.250:23686;FAIL;
+IN;43.224.10.35:6667;FAIL;
+US;173.236.185.96:22960;FAIL;
+AR;181.3.37.213:1080;200.73.130.62;0:00:08.508165
+AR;181.3.49.28:1080;FAIL;
+US;173.236.191.119:22960;173.236.184.139;0:00:08.729647
+BR;181.3.67.154:1080;FAIL;
+US;104.248.0.141:57391;104.248.0.141;0:00:03.865643
+AR;186.152.149.227:1080;200.73.130.62;0:00:03.071001
+CA;186.126.82.88:1080;FAIL;
+AR;186.126.151.73:1080;200.73.130.62;0:00:05.884195
+DE;173.212.201.250:54349;FAIL;
+AR;181.0.5.196:1080;FAIL;
+NL;142.93.137.235:6191;142.93.137.235;0:00:04.257492
+SG;206.189.158.28:4454;FAIL;
+US;157.230.154.211:32381;157.230.154.211;0:00:10.416110
+AR;186.126.49.178:1080;FAIL;
+CA;181.3.40.39:1080;FAIL;
+US;95.217.132.133:3598;FAIL;
+FR;51.68.134.249:30204;FAIL;
+US;104.238.111.167:53308;FAIL;
+DE;171.221.35.24:1080;FAIL;
+NL;188.166.104.152:44924;FAIL;
+SG;129.226.196.49:13181;129.226.196.49;0:00:13.210261
+AR;181.3.55.161:1080;FAIL;
+HK;101.132.120.74:1080;FAIL;
+SE;95.217.132.133:3508;FAIL;
+CA;186.126.129.149:1080;51.79.52.142;0:00:08.184306
+AR;181.101.19.224:1080;FAIL;
+AR;181.3.7.234:1080;FAIL;
+AR;181.6.28.131:1080;FAIL;
+BR;181.6.114.165:1080;209.14.2.57;0:00:18.254419
+DE;173.212.201.250:33464;FAIL;
+NL;146.185.132.87:55158;FAIL;
+HK;150.109.148.234:1234;FAIL;
+HU;85.90.161.117:2021;85.90.161.117;0:00:02.127226
+AR;181.5.201.229:1080;200.73.132.119;0:00:15.974410
+US;72.49.49.11:31034;FAIL;
+US;97.74.6.64:45683;FAIL;
+US;186.126.95.145:10808;FAIL;
+DE;54.38.157.22:9999;54.38.157.22;0:00:08.000757
+FR;51.68.134.250:30204;FAIL;
+BR;186.126.89.33:1080;FAIL;
+FR;51.68.134.242:30204;FAIL;
+US;166.62.85.184:42828;166.62.85.184;0:00:04.136324
+US;173.236.186.172:22960;173.236.184.139;0:00:04.403408
+BR;181.83.228.40:1080;FAIL;
+US;165.22.13.68:15576;165.22.13.68;0:00:04.907470
+US;104.248.48.169:30588;FAIL;
+SG;206.189.92.74:38888;FAIL;
+AR;181.3.63.142:1080;FAIL;
+AR;186.126.87.224:1080;FAIL;
+BR;181.3.46.205:1080;FAIL;
+CA;181.3.16.31:1080;FAIL;
+SG;45.76.187.35:36600;FAIL;
+US;173.236.186.230:22960;173.236.184.139;0:00:03.272663
+RU;171.221.44.248:1080;FAIL;
+US;181.7.201.96:1080;FAIL;
+US;147.135.116.172:55546;147.135.116.172;0:00:05.626279
+AR;181.3.29.244:1080;FAIL;
+BR;186.152.147.113:1080;FAIL;
+AR;181.102.81.144:1080;FAIL;
+US;104.248.0.141:30247;104.248.0.141;0:00:01.176155
+US;104.238.212.43:1080;104.238.215.49;0:00:05.161615
+BR;186.152.26.161:1080;191.252.102.212;0:00:09.528139
+US;143.110.153.171:3240;FAIL;
+PS;213.6.61.150:9999;FAIL;
+IN;43.224.10.30:6667;FAIL;
+AR;181.101.4.206:1080;FAIL;
+SG;206.189.158.28:48500;FAIL;
+FR;54.36.246.232:11380;54.36.246.232;0:00:07.263434
+DE;213.136.89.190:4374;FAIL;
+SG;206.189.158.28:64028;FAIL;
+CL;186.126.131.207:1080;FAIL;
+IN;43.224.10.36:6667;43.224.10.36;0:00:08.870324
+CA;181.3.93.39:1080;FAIL;
+NL;142.93.138.78:63421;142.93.138.78;0:00:02.779517
+NL;146.185.132.87:49041;146.185.132.87;0:00:08.279986
+DE;95.217.132.133:3008;FAIL;
+BR;181.101.52.45:1080;FAIL;
+US;192.169.201.24:51100;FAIL;
+BR;181.3.24.19:1080;FAIL;
+AR;186.126.15.57:1080;FAIL;
+PL;5.226.69.12:42717;FAIL;
+DE;213.136.89.190:13492;FAIL;
+BR;181.102.141.53:1080;FAIL;
+US;74.208.102.54:31200;74.208.102.54;0:00:10.078336
+GB;95.217.132.133:3273;FAIL;
+CA;159.203.42.128:28393;159.203.42.128;0:00:04.454060
+BR;181.101.29.81:1080;FAIL;
+CA;181.3.84.102:1080;FAIL;
+US;173.236.189.156:7595;173.236.184.102;0:00:06.332096
+FR;51.68.134.246:30204;FAIL;
+BR;181.6.24.228:1080;FAIL;
+US;95.217.132.133:3503;FAIL;
+AR;186.126.54.106:1080;FAIL;
+SG;206.189.158.28:48751;FAIL;
+NL;178.62.136.189:51423;178.62.136.189;0:00:06.756095
+US;173.236.187.42:22960;173.236.184.139;0:00:07.256691
+IN;43.224.10.46:6667;FAIL;
+US;206.189.231.206:2106;FAIL;
+SG;95.217.132.133:3286;FAIL;
+SG;129.226.196.49:22157;129.226.196.49;0:00:09.336891
+US;173.236.186.241:22960;173.236.184.139;0:00:02.345419
+HK;119.28.81.177:20412;FAIL;
+RU;31.25.243.40:9432;FAIL;
+CA;181.3.65.57:1080;51.222.13.156;0:00:06.097943
+AR;181.0.16.160:1080;FAIL;
+UA;31.128.248.2:1080;FAIL;
+HK;36.150.108.65:1080;FAIL;
+RU;31.25.243.40:9159;FAIL;
+US;181.0.26.16:1080;FAIL;
+CA;181.6.61.241:1080;FAIL;
+FR;51.68.134.243:25623;51.68.134.240;0:00:02.797034
+BR;181.3.56.31:1080;20.195.214.142;0:00:05.865545
+US;147.135.116.172:47283;147.135.116.172;0:00:07.138716
+SG;113.77.87.43:1081;FAIL;
+FR;51.68.134.255:25623;51.68.134.240;0:00:02.196854
+IN;103.216.82.37:6667;103.216.82.37;0:00:04.271719
+HK;223.199.179.145:1080;FAIL;
+US;104.238.111.167:29182;104.238.111.167;0:00:07.471943
+GB;46.101.56.138:33232;FAIL;
+DE;213.136.89.190:51808;213.136.89.190;0:00:01.532093
+NL;142.93.137.235:1429;142.93.137.235;0:00:04.408165
+BR;181.5.210.85:1080;FAIL;
+US;67.227.193.162:24595;67.227.193.162;0:00:07.794617
+FR;51.68.134.248:25623;51.68.134.240;0:00:07.714408
+HK;153.37.113.125:1080;42.3.24.58;0:00:03.530263
+US;104.248.0.141:23668;104.248.0.141;0:00:01.404311
+AR;186.126.84.156:1080;200.73.128.105;0:00:15.717142
+PH;210.16.73.81:1080;FAIL;
+FR;51.68.134.252:30204;FAIL;
+CA;181.5.242.212:1080;FAIL;
+AR;181.6.14.34:1080;FAIL;
+NL;146.185.132.87:59746;FAIL;
+SG;206.189.158.28:15615;FAIL;
+GB;159.65.26.54:34787;159.65.26.54;0:00:07.312364
+FR;51.68.134.254:25623;51.68.134.240;0:00:09.785792
+SG;206.189.158.28:11007;FAIL;
+AR;186.152.26.173:1080;FAIL;
+US;206.189.231.206:53323;FAIL;
+US;192.169.201.24:7495;FAIL;
+AR;181.101.57.210:1080;FAIL;
+US;173.236.184.50:7595;FAIL;
+US;181.7.211.6:1080;FAIL;
+AR;186.126.80.109:1080;FAIL;
+CA;181.3.67.17:1080;FAIL;
+US;165.22.13.68:25327;165.22.13.68;0:00:06.029895
+CA;159.203.42.128:47524;159.203.42.128;0:00:09.931594
+AR;181.101.57.64:1080;200.73.133.154;0:00:12.503640
+BR;181.6.134.15:1080;FAIL;
+AR;181.6.35.81:1080;FAIL;
+US;173.236.186.228:22960;FAIL;
+CA;181.102.111.148:1080;FAIL;
+US;181.3.39.201:1080;FAIL;
+DE;95.217.132.133:3412;FAIL;
+US;206.189.231.206:50825;206.189.231.206;0:00:01.618712
+SG;206.189.158.28:47419;FAIL;
+DE;45.149.76.184:9051;FAIL;
+GB;159.65.26.54:2975;FAIL;
+US;64.34.216.68:40741;64.34.205.58;0:00:13.192013
+US;173.236.188.107:7595;173.236.184.102;0:00:03.604567
+US;166.62.85.224:42790;FAIL;
+DE;181.101.10.10:1080;78.47.73.135;0:00:20.253722
+RU;95.107.37.109:3109;85.26.186.44;0:00:04.610048
+AR;181.3.29.168:1080;FAIL;
+AR;181.6.128.215:1080;FAIL;
+US;95.217.132.133:3132;FAIL;
+AR;186.126.120.70:1080;FAIL;
+UA;80.73.9.238:1080;FAIL;
+IN;43.224.10.42:6667;43.224.10.42;0:00:12.005869
+US;206.189.180.62:7934;FAIL;
+AR;181.3.52.116:1080;FAIL;
+AR;181.3.91.214:1080;FAIL;
+DE;213.136.89.190:56844;FAIL;
+BR;181.7.198.151:1080;191.252.113.106;0:00:15.269279
+US;104.248.0.141:54251;FAIL;
+GB;176.58.100.26:27016;FAIL;
+HK;113.240.216.243:1080;FAIL;
+AR;186.126.66.41:1080;FAIL;
+US;173.236.189.250:7595;FAIL;
+BR;181.5.230.16:1080;191.252.113.106;0:00:19.073131
+US;50.62.35.16:29643;FAIL;
+IN;103.21.163.76:6667;103.21.163.76;0:00:08.463147
+DK;65.21.49.222:9174;FAIL;
+US;104.238.97.215:7772;FAIL;
+AR;181.3.68.52:1080;FAIL;
+AR;186.126.92.77:1080;FAIL;
+US;95.217.132.133:3141;FAIL;
+BR;186.126.168.161:1080;FAIL;
+DE;46.4.156.212:18588;FAIL;
+SG;206.189.158.28:7476;FAIL;
+AR;181.6.114.157:1080;200.73.132.187;0:00:13.969104
+US;181.102.141.210:1080;FAIL;
+BR;181.5.212.118:1080;FAIL;
+SG;45.76.187.35:44560;45.76.187.35;0:00:09.819446
+AR;186.152.150.124:1080;FAIL;
+AR;186.126.141.216:1080;FAIL;
+CA;186.152.114.192:1080;FAIL;
+US;173.236.191.150:22960;173.236.184.139;0:00:09.824398
+AR;181.7.207.196:1080;FAIL;
+JP;138.91.19.96:1953;138.91.19.96;0:00:12.281648
+CL;186.126.48.110:1080;FAIL;
+US;74.208.101.185:44614;74.208.102.54;0:00:10.538888
+AR;181.101.53.210:1080;FAIL;
+US;65.21.49.222:9270;FAIL;
+US;173.236.189.19:22960;FAIL;
+US;95.217.132.133:3137;FAIL;
+AR;186.126.42.157:1080;FAIL;
+US;173.236.189.188:22960;173.236.184.139;0:00:05.774545
+US;8.210.163.246:50001;FAIL;
+DE;213.136.89.190:5136;FAIL;
+US;173.236.186.235:22960;173.236.184.139;0:00:04.093197
+AR;186.126.176.41:1080;FAIL;
+US;173.236.189.191:22960;173.236.184.139;0:00:02.491511
+US;173.236.188.227:7595;FAIL;
+SG;206.189.158.28:21471;206.189.158.28;0:00:07.368676
+US;95.217.132.133:3463;FAIL;
+US;173.236.186.1:22960;FAIL;
+BR;186.126.101.194:1080;FAIL;
+AR;181.101.48.228:1080;FAIL;
+US;95.217.132.133:3443;FAIL;
+HK;119.28.81.177:59430;FAIL;
+AR;181.3.27.242:10808;FAIL;
+AR;181.0.30.128:1080;FAIL;
+US;173.236.186.167:22960;173.236.184.139;0:00:06.491244
+PL;5.226.69.12:46975;5.226.69.12;0:00:07.836800
+NL;142.93.137.235:38902;142.93.137.235;0:00:02.734874
+US;173.236.188.12:7595;FAIL;
+DE;213.136.89.190:52010;FAIL;
+US;173.236.188.156:7595;FAIL;
+BR;181.3.36.182:1080;FAIL;
+FR;51.75.42.92:25623;51.68.134.240;0:00:09.193243
+US;173.236.189.132:22960;173.236.184.139;0:00:03.395630
+US;173.236.185.29:22960;FAIL;
+AR;186.126.50.32:1080;FAIL;
+RU;31.25.243.40:9261;FAIL;
+IR;5.56.134.237:45698;5.56.134.237;0:00:02.117181
+ID;103.224.103.116:1080;FAIL;
+CN;110.90.223.72:57114;FAIL;
+CN;59.61.160.63:16790;FAIL;
+CN;119.187.146.163:1080;119.187.146.163;0:00:03.103184
+CN;59.61.160.179:16790;59.61.160.179;0:00:02.662718
+CN;113.123.0.217:1080;FAIL;
+CN;111.1.36.135:9053;115.238.101.42;0:00:02.840302
+CN;111.1.36.132:9053;115.238.101.39;0:00:04.400966
+CN;3.131.207.170:11098;FAIL;
+CN;117.174.160.105:1080;117.174.160.105;0:00:05.121497
+CN;36.27.223.80:35880;120.33.231.36;0:00:05.118523
+CN;60.168.25.143:4216;FAIL;
+CN;47.104.16.8:6667;FAIL;
+CN;114.236.90.5:1080;FAIL;
+ID;139.255.89.4:1080;139.255.89.2;0:00:03.779929
+CN;111.225.153.226:57114;FAIL;
+CN;134.175.90.111:8889;FAIL;
+CN;111.1.36.132:9055;FAIL;
+CN;121.206.250.10:57114;121.206.250.10;0:00:02.844775
+CN;42.193.148.214:1080;FAIL;
+CN;111.1.36.134:9053;115.238.101.41;0:00:03.048803
+CN;39.96.175.55:1080;FAIL;
+CN;47.92.252.178:3129;FAIL;
+CN;122.152.219.54:57164;122.152.219.54;0:00:06.862155
+ID;36.89.86.49:56845;36.89.86.49;0:00:03.790392
+CN;36.27.223.80:35101;FAIL;
+CN;36.27.223.80:34638;106.114.146.84;0:00:05.630091
+CN;218.64.122.99:7302;218.64.122.99;0:00:09.361461
+ID;36.94.126.50:1080;36.94.126.50;0:00:05.162022
+CN;47.100.19.147:3129;47.100.19.147;0:00:11.339600
+CN;122.152.219.54:1749;FAIL;
+CN;59.61.160.153:16790;59.61.160.153;0:00:08.683302
diff --git a/facebook/start_database.py b/facebook/start_database.py
new file mode 100644
index 0000000..30d3dc3
--- /dev/null
+++ b/facebook/start_database.py
@@ -0,0 +1,23 @@
+from arango import ArangoClient
+from getpass import getpass
+from config import *
+from time import sleep
+
+for i in range(0, 6, 1):
+ if i == 5:
+ exit()
+ try:
+ # Om scriptet körs på Macbook finns lösenordet i en fil
+ with open("../password_arango.txt") as f:
+ pwd = f.readline()
+ except FileNotFoundError:
+ if pwd == None:
+ pwd = getpass(f'Lösenord för {user_arango}: ')
+
+ try:
+ db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)
+ db.collection('members').random() # För att testa löseordet/kopplingen.
+ break
+ except:
+ print("Fel lösenord.")
+ sleep(1)
diff --git a/facebook/stats.py b/facebook/stats.py
new file mode 100644
index 0000000..bcf653f
--- /dev/null
+++ b/facebook/stats.py
@@ -0,0 +1,74 @@
+from datetime import datetime
+from getpass import getpass
+from time import sleep
+
+from arango import ArangoClient
+from json2html import json2html
+
+
+def now():
+ """ Returns current date and time as string"""
+ return datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+
+def write_stats(db, continuous=False):
+ while True:
+ d = {}
+ for col in db.collections():
+ if not col['system']:
+ d[col['name']] = db.collection(col['name']).count()
+ del d['stats']
+ #d['time'] = now()
+ cursor = db.aql.execute(
+ """
+ FOR doc IN members
+ FILTER doc.checked == true
+ COLLECT WITH COUNT INTO length
+ RETURN length
+ """
+ )
+ d['checked_members'] = cursor.next()
+
+
+ # Hur många konton per säljare som finns kvar
+ cursor = db.aql.execute(
+ '''
+ for doc in profiles
+ filter has(doc, "vendor")
+ COLLECT vendor = doc.vendor WITH COUNT INTO length
+ RETURN {
+ "vendor" : vendor,
+ "active" : length
+ }
+ ''')
+ d['active_vendors'] = [doc for doc in cursor]
+
+ d['_key'] = now()[:13]
+ db.insert_document( "stats", d, overwrite=True)
+
+ # Skriv en html-fil
+ with open('website/fb-webbapp/stats.html', 'a+') as html:
+ html.truncate(0)
+ html.write('
')
+
+ html.write(json2html.convert(json = d))
+
+ # Sov för att fortsätta senare
+ if continuous:
+ sleep(86400)
+ else:
+ break
+
+# Info för arangodb
+user_arango = "Lasse"
+db_arango = "facebook"
+host_arango = "http://192.168.0.4:8529"
+
+# Starta koppling till arangodb
+# Avkryptera lösen till arango
+pwd = getpass(f'Arangolösenord för {user_arango}:').strip()
+
+db = ArangoClient(hosts=host_arango).db(
+ db_arango, username=user_arango, password=pwd
+)
+
+write_stats(db, continuous=True)
diff --git a/facebook/testclass.py b/facebook/testclass.py
new file mode 100644
index 0000000..1907a00
--- /dev/null
+++ b/facebook/testclass.py
@@ -0,0 +1,131 @@
+class Profile:
+ def __init__(self, profile, container, proxieservers):
+ """Creates a new profile to do searches with.
+
+ Args:
+ profile (dict): Document fetched from database.
+ container (str): Docker container that runs the script.
+ """
+
+ self.doc = profile
+
+ # Användaruppgifter
+ self.name = self.doc["name"].strip()
+ self.email = self.doc["email"]
+ self.pwd = self.doc["pwd"]
+ self.server = self.doc["server"]
+ self.cookie = self.doc["cookie"]
+ self.useragent = self.doc["useragent"]
+
+ self.proxieservers = proxieservers
+ self.blocked = False
+ self.container = str(container)
+ self.users_checked = 0
+
+ # Ange proxies
+ session = requests.Session()
+ session.proxies = self.doc['proxies']
+
+ # Starta browser
+ user_agent = self.useragent
+ self.browser = RoboBrowser(
+ session=session, user_agent=user_agent, history=False, parser="lxml"
+ )
+ try:
+ self.browser.session.cookies = pickle.load(
+ open("data/cookie_{}.pkl".format(self.name), "rb")
+ )
+ self.logged_in = True
+ except:
+ try:
+ self.browser.session.cookies.update(self.cookie)
+ self.logged_in = True
+ except:
+ self.logged_in = False
+
+ def update_time(self):
+ """ Uppdatera dokumentet i arango. """
+ self.doc["in_use"] = nowstamp()
+ db.update_document(self.doc, check_rev=False)
+
+ def viewing(self):
+ """ Returnerar browser i html-format """
+ return self.browser.parsed
+
+ def open(self, url):
+ n = 0
+ while True:
+ n += 1
+ sleep(1)
+ try:
+ self.browser.open(url)
+ if '/a/nux/wizard/nav.php?step=phone&skip' in self.viewing():
+ self.browser.open(url_bas + '/a/nux/wizard/nav.php?step=phone&skip')
+ break
+ except Exception as e:
+ print(e)
+ print(n)
+ _print(self, None, f'Kunde inte öppna url {url}')
+ if n == 5:
+ if 'Connection refused' in e:
+ self.doc['e'] = e
+ db.insert_document('blocked_profiles', self.doc)
+ n = 0
+ from arangodb import get_profile, remove_profile
+ # Ta bort den gamla profilen från databasen och ersätt profile med nytt innehåll från ny profil
+ remove_profile(self)
+ self.__init__(get_profile(self.proxieservers), self.container)
+ _print(self, None, f'Ny profil hämtad {self.email}')
+ self.update_time()
+ else:
+ sleep(40)
+
+ def accept_cookies(self):
+ """ Accepterar cookies """
+ self.browser.open("https://mbasic.facebook.com")
+ soup = BeautifulSoup(str(self.browser.parsed), "lxml")
+ if "accept all" not in soup.text.lower():
+ sleep_(2)
+ cookie_accept_url = "https://mbasic.facebook.com/cookie/consent-page"
+ self.browser.open(cookie_accept_url)
+ sleep_(2)
+ try:
+ form = self.browser.get_form()
+ self.browser.submit_form(form)
+ _print(self, None, f"Accepterade cookies för {self.name}")
+ sleep_(2)
+ update_cookie(self.browser.session.cookies, self)
+ except:
+ try:
+ write_error(12, self, soup=self.browser.parsed)
+ except:
+ pass
+ _print(self, None, f"Accepterade inte cookies för {self.name}")
+ def login(self):
+ """ Loggar in på Facebook. """
+
+ print("Loggar in {}".format(self.name))
+
+ # Gå till log in-sidan
+ self.browser.open("https://mbasic.facebook.com/login")
+
+ # Kolla om browser redan är inloggad
+ soup = BeautifulSoup(str(self.browser.parsed), "lxml")
+ if "log out" in soup.text.lower():
+ print("Redan inloggad.")
+ try:
+ # Hitta och fyll i formulär
+ form = self.browser.get_form(id="login_form")
+ form["email"].value = self.email
+ form["pass"].value = self.pwd
+ self.browser.submit_form(form, submit=form["login"])
+ # Vänta lite och uppdatera cookie
+ print("Loggade in.")
+ sleep_(2)
+ self.open(url_bas)
+ sleep_(2)
+ except TypeError:
+ try:
+ write_error(11, self, soup=soup, profile=self.name)
+ except:
+ pass
\ No newline at end of file
diff --git a/fb-webbapp/main.py b/fb-webbapp/main.py
new file mode 100644
index 0000000..8a562e5
--- /dev/null
+++ b/fb-webbapp/main.py
@@ -0,0 +1,14 @@
+from flask import Flask, render_template
+import json
+from json2html import json2html
+
+app = Flask(__name__)
+
+
+@app.route("/")
+def stats():
+ return render_template("stats.html")
+
+if __name__ == "__main__":
+ app.run(debug=True)
+
diff --git a/integrity.py b/integrity.py
new file mode 100644
index 0000000..3aad5db
--- /dev/null
+++ b/integrity.py
@@ -0,0 +1,15 @@
+import subprocess
+import requests
+from time import sleep
+
+subprocess.run(['sudo', 'wg-quick', 'down', 'integrity'])
+
+with open('ip.txt', 'a+') as f:
+ while True:
+ subprocess.run(['wg-quick', 'up', 'integrity'] )
+ sleep(5)
+ ip = requests.get('https://api.ipify.org').text
+ print(ip)
+ f.write(f'{ip}\n')
+ subprocess.run(['wg-quick', 'down', 'integrity'])
+ sleep(5)
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 974731b..47443c8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@ idna==2.10
lxml==4.6.2
pycparser==2.20
PyJWT==2.0.1
-PyNaCl==1.4.0
+#PyNaCl==1.4.0
PySocks==1.7.1
python-arango==7.1.0
requests==2.25.1
@@ -18,3 +18,4 @@ soupsieve==2.2
toml==0.10.2
urllib3==1.26.3
Werkzeug==1.0.1
+json2html
diff --git a/workspace.code-workspace b/workspace.code-workspace
index d8e064d..189cad0 100644
--- a/workspace.code-workspace
+++ b/workspace.code-workspace
@@ -1,12 +1,17 @@
{
- "folders": [
- {
- "path": "."
- },
- {
- "path": "facebook"
- }
- ],
- "settings": {
- }
+ "folders": [
+ {
+ "path": "."
+ },
+ {
+ "path": "../mrkoll"
+ },
+ {
+ "path": "facebook"
+ }
+ ],
+ "settings": {
+ "python.pythonPath": "/Users/Lasse/Datorgemensamt/Programmeringsprojekt/Facebook/fb-scraper/.venv/bin/python"
+ },
+
}
\ No newline at end of file