You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
309 lines
9.6 KiB
309 lines
9.6 KiB
import random |
|
from datetime import datetime |
|
from time import sleep |
|
|
|
import requests |
|
import werkzeug |
|
from bs4 import BeautifulSoup |
|
|
|
werkzeug.cached_property = werkzeug.utils.cached_property |
|
from robobrowser import RoboBrowser |
|
|
|
from arangodb import db |
|
from config import url_bas |
|
from helpers import sleep_, update_cookie, write_error, nowstamp, _print |
|
|
|
|
|
class User: |
|
def __init__(self, username, mode, other_pictures=[]): |
|
self.collection = "members" |
|
self.username = str(username) |
|
self.mode = mode |
|
self.fetched = datetime.now().strftime("%Y%m%d") |
|
self.url_coverphotos = "" |
|
self.id = "" |
|
self.url_likes = "" |
|
self.url_about = "" |
|
self.url_timeline = "" |
|
self.url_album = "" |
|
self.url_profilepictures = "" |
|
self.profile_pictures = 0 |
|
self.pictures = [] |
|
self.url_friends = "" |
|
self.url = "" |
|
self.name = "" |
|
self.url_other_pictures = other_pictures |
|
self.reactions = 0 |
|
self.profile_pictures = 0 |
|
self.checked_pictures = [] |
|
|
|
def add_to_db(self): |
|
# Lägg till profilen till arrango |
|
db.insert_document( |
|
self.collection, |
|
{ |
|
"_key": self.username, |
|
"url": self.url, |
|
"name": self.name, |
|
"profile_pictures": self.profile_pictures, |
|
"facebook_id": self.id, |
|
"timeline": self.url_timeline, |
|
"likes": self.url_likes, |
|
"about": self.url_about, |
|
"cover photos": self.url_coverphotos, |
|
"fetched": self.fetched, |
|
"reactions": self.reactions, |
|
"mode": self.mode, |
|
"pictures": self.pictures, |
|
}, |
|
overwrite_mode="update", |
|
silent=True, |
|
keep_none=False, |
|
) |
|
|
|
def checked(self): |
|
db.update_document( |
|
{ |
|
"_id": "members/" + str(self.username), |
|
"checked": True, |
|
"pictures_checked": len(self.checked_pictures), |
|
"checked_pictures": self.checked_pictures, |
|
"reactions": self.reactions, |
|
} |
|
) |
|
|
|
|
|
class Picture: |
|
def __init__(self, user): |
|
self.collection = "pictures" |
|
self.user = user |
|
self.id = "" |
|
self.url_full = "" |
|
self.date = "" |
|
self.url = "" |
|
self.no_reactions = "" |
|
self.reactions = [] |
|
self.src = "" |
|
|
|
|
|
def add_to_db(self): |
|
db.insert_document( |
|
self.collection, |
|
{ |
|
"_key": self.id, |
|
"url": self.url_full, |
|
"date": self.date, |
|
"url": self.url, |
|
"no_reactions": self.no_reactions, |
|
"user": self.user, |
|
"src": self.src, |
|
}, |
|
overwrite_mode="update", |
|
silent=True, |
|
keep_none=False, |
|
) |
|
|
|
|
|
class Profile: |
|
def __init__(self, profile, container, proxieservers): |
|
"""Creates a new profile to do searches with. |
|
|
|
Args: |
|
profile (dict): Document fetched from database. |
|
container (str): Docker container that runs the script. |
|
""" |
|
|
|
self.doc = profile |
|
|
|
# Användaruppgifter |
|
self.name = self.doc["name"].strip() |
|
self.email = self.doc["email"] |
|
self.pwd = self.doc["pwd"] |
|
self.server = self.doc["server"] |
|
self.cookie = self.doc["cookie"] |
|
self.useragent = self.doc["useragent"] |
|
|
|
self.proxieservers = proxieservers |
|
self.blocked = False |
|
self.container = str(container) |
|
self.users_checked = 0 |
|
self.write = True |
|
|
|
# Ange proxies |
|
session = requests.Session() |
|
session.proxies = self.doc["proxies"] |
|
|
|
# Starta browser |
|
user_agent = self.useragent |
|
self.browser = RoboBrowser( |
|
session=session, user_agent=user_agent, history=False, parser="lxml", timeout=20 |
|
) #TODO Kolla timeout på den här |
|
|
|
# TODO Ta bort gamla metoden om nya (hämta från doc) fungerar |
|
# try: |
|
# # Försök hämta cookie från fil |
|
# self.browser.session.cookies = pickle.load( |
|
# open("data/cookie_{}.pkl".format(self.name), "rb") |
|
# ) |
|
# self.logged_in = True |
|
|
|
try: |
|
self.browser.session.cookies.update(self.cookie) |
|
self.logged_in = True |
|
except: |
|
self.logged_in = False |
|
|
|
def update_time(self): |
|
"""Uppdatera dokumentet i arango.""" |
|
self.doc["in_use"] = nowstamp() |
|
db.update_document(self.doc, check_rev=False) |
|
|
|
def viewing(self): |
|
"""Returnerar browser i html-format""" |
|
return self.browser.parsed |
|
|
|
def open(self, url): |
|
n = 0 |
|
while True: |
|
n += 1 |
|
sleep(1) |
|
try: |
|
# Försök öppna url, om det misslyckas så vänta lite och försök sen igen |
|
self.browser.open(url) |
|
if "/a/nux/wizard/nav.php?step=phone&skip" in self.viewing(): |
|
self.browser.open( |
|
url_bas + "/a/nux/wizard/nav.php?step=phone&skip" |
|
) |
|
break |
|
except Exception as e: |
|
print(e) |
|
print(n) |
|
_print(self, None, f"Kunde inte öppna url {url}") |
|
if n == 5: |
|
if "Connection refused" in e: |
|
self.doc["e"] = e |
|
db.insert_document("blocked_profiles", self.doc) |
|
n = 0 |
|
from arangodb import get_profile, remove_profile |
|
|
|
# Ta bort den gamla profilen från databasen och ersätt profile med nytt innehåll från ny profil |
|
remove_profile(self) |
|
self.__init__(get_profile(self.proxieservers), self.container) |
|
_print(self, None, f"Ny profil hämtad {self.email}") |
|
self.update_time() |
|
else: |
|
sleep(40) |
|
|
|
def accept_cookies(self): |
|
"""Accepterar cookies""" |
|
self.browser.open("https://mbasic.facebook.com") |
|
soup = BeautifulSoup(str(self.browser.parsed), "lxml") |
|
if "accept all" not in soup.text.lower(): |
|
sleep_(2) |
|
cookie_accept_url = "https://mbasic.facebook.com/cookie/consent-page" |
|
self.browser.open(cookie_accept_url) |
|
sleep_(2) |
|
try: |
|
form = self.browser.get_form() |
|
self.browser.submit_form(form) |
|
_print(self, None, f"Accepterade cookies för {self.name}") |
|
sleep_(2) |
|
update_cookie(self.browser.session.cookies, self) |
|
except: |
|
try: |
|
write_error(12, self, soup=self.browser.parsed) |
|
except: |
|
pass |
|
_print(self, None, f"Accepterade inte cookies för {self.name}") |
|
|
|
def login(self): |
|
"""Loggar in på Facebook.""" |
|
|
|
print("Loggar in {}".format(self.name)) |
|
|
|
# Gå till log in-sidan |
|
self.browser.open("https://mbasic.facebook.com/login") |
|
|
|
# Kolla om browser redan är inloggad |
|
soup = BeautifulSoup(str(self.browser.parsed), "lxml") |
|
if "log out" in soup.text.lower(): |
|
print("Redan inloggad.") |
|
try: |
|
# Hitta och fyll i formulär |
|
form = self.browser.get_form(id="login_form") |
|
form["email"].value = self.email |
|
form["pass"].value = self.pwd |
|
self.browser.submit_form(form, submit=form["login"]) |
|
# Vänta lite och uppdatera cookie |
|
print("Loggade in.") |
|
sleep_(2) |
|
self.open(url_bas) |
|
|
|
sleep_(2) |
|
except TypeError: |
|
try: |
|
write_error(11, self, soup=soup, profile=self.name) |
|
except: |
|
print('KUNDE INTE LOGGA IN') |
|
pass |
|
|
|
def update_cookie(self, cookie): |
|
self.cookie = cookie |
|
db.update_document({"_id": self.doc["_id"], "cookie": cookie}, check_rev=False) |
|
|
|
|
|
class Proxies: |
|
def __init__(self): |
|
self.proxies = [ |
|
"gb25-wg.socks5.mullvad.net:1080", |
|
"gb26-wg.socks5.mullvad.net:1080", |
|
"gb27-wg.socks5.mullvad.net:1080", |
|
"gb28-wg.socks5.mullvad.net:1080", |
|
"gb29-wg.socks5.mullvad.net:1080", |
|
] |
|
|
|
def get_proxie(self): |
|
return self.proxies.pop(random.randrange(0, len(self.proxies), 1)) |
|
|
|
|
|
class Friend: |
|
def __init__(self, user): |
|
self.collection = "members" |
|
self.user = user # The friends friend |
|
self.username = "" |
|
self.url = "" |
|
self.name = "" |
|
self.id = "" |
|
|
|
def add_to_db(self): |
|
db.insert_document( |
|
self.collection, |
|
{ |
|
"_key": str(self.username), |
|
"url": url_bas + self.url, |
|
"name": self.name, |
|
'id_from_seemore_url': self.id |
|
}, |
|
overwrite_mode="update", |
|
silent=True, |
|
) |
|
|
|
|
|
class Reaction: |
|
def __init__(self, user, friend_username, picture_id): |
|
self.collection = "picture_reactions" |
|
self.user = user |
|
self.picture_id = picture_id |
|
self.user_name_friend = friend_username |
|
self.type = False |
|
|
|
def get_dict(self): |
|
key = str(self.picture_id) + "_" + str(self.user_name_friend) |
|
return { |
|
"_to": "members/" + str(self.user), |
|
"_from": "members/" + str(self.user_name_friend), |
|
"_key": key, |
|
"_id": "picture_reactions/" + key, |
|
"picture": self.picture_id, |
|
"reaction": self.type, |
|
}
|
|
|