import random from datetime import datetime from time import sleep import requests import werkzeug from bs4 import BeautifulSoup werkzeug.cached_property = werkzeug.utils.cached_property from robobrowser import RoboBrowser from arangodb import db from config import url_bas from helpers import sleep_, update_cookie, write_error, nowstamp, _print class User: def __init__(self, username, mode, other_pictures=[]): self.collection = "members" self.username = str(username) self.mode = mode self.fetched = datetime.now().strftime("%Y%m%d") self.url_coverphotos = "" self.id = "" self.url_likes = "" self.url_about = "" self.url_timeline = "" self.url_profilepictures = "" self.profile_pictures = 0 self.pictures = [] self.url_friends = "" self.url = "" self.name = "" self.url_other_pictures = other_pictures self.reactions = 0 self.profile_pictures = 0 self.checked_pictures = [] def add_to_db(self): # Lägg till profilen till arrango db.insert_document( self.collection, { "_key": self.username, "url": self.url, "name": self.name, "profile_pictures": self.profile_pictures, "facebook_id": self.id, "timeline": self.url_timeline, "likes": self.url_likes, "about": self.url_about, "cover photos": self.url_coverphotos, "fetched": self.fetched, "reactions": self.reactions, "mode": self.mode, "pictures": self.pictures, }, overwrite_mode="update", silent=True, keep_none=False, ) def checked(self): db.update_document( { "_id": "members/" + str(self.username), "checked": True, "pictures_checked": len(self.checked_pictures), "checked_pictures": self.checked_pictures, "reactions": self.reactions, } ) class Picture: def __init__(self, user): self.collection = "pictures" self.user = user self.id = "" self.url_full = "" self.date = "" self.url = "" self.no_reactions = "" self.reactions = [] self.src = "" def add_to_db(self): db.insert_document( self.collection, { "_key": self.id, "url": self.url_full, "date": self.date, "url": self.url, "no_reactions": self.no_reactions, "user": self.user, "src": self.src, }, overwrite_mode="update", silent=True, keep_none=False, ) class Profile: def __init__(self, profile, container, proxieservers): """Creates a new profile to do searches with. Args: profile (dict): Document fetched from database. container (str): Docker container that runs the script. """ self.doc = profile # Användaruppgifter self.name = self.doc["name"].strip() self.email = self.doc["email"] self.pwd = self.doc["pwd"] self.server = self.doc["server"] self.cookie = self.doc["cookie"] self.useragent = self.doc["useragent"] self.proxieservers = proxieservers self.blocked = False self.container = str(container) self.users_checked = 0 self.write = True # Ange proxies session = requests.Session() session.proxies = self.doc["proxies"] # Starta browser user_agent = self.useragent self.browser = RoboBrowser( session=session, user_agent=user_agent, history=False, parser="lxml", timeout=20 ) #TODO Kolla timeout på den här # TODO Ta bort gamla metoden om nya (hämta från doc) fungerar # try: # # Försök hämta cookie från fil # self.browser.session.cookies = pickle.load( # open("data/cookie_{}.pkl".format(self.name), "rb") # ) # self.logged_in = True try: self.browser.session.cookies.update(self.cookie) self.logged_in = True except: self.logged_in = False def update_time(self): """Uppdatera dokumentet i arango.""" self.doc["in_use"] = nowstamp() db.update_document(self.doc, check_rev=False) def viewing(self): """Returnerar browser i html-format""" return self.browser.parsed def open(self, url): n = 0 while True: n += 1 sleep(1) try: # Försök öppna url, om det misslyckas så vänta lite och försök sen igen self.browser.open(url) if "/a/nux/wizard/nav.php?step=phone&skip" in self.viewing(): self.browser.open( url_bas + "/a/nux/wizard/nav.php?step=phone&skip" ) break except Exception as e: print(e) print(n) _print(self, None, f"Kunde inte öppna url {url}") if n == 5: if "Connection refused" in e: self.doc["e"] = e db.insert_document("blocked_profiles", self.doc) n = 0 from arangodb import get_profile, remove_profile # Ta bort den gamla profilen från databasen och ersätt profile med nytt innehåll från ny profil remove_profile(self) self.__init__(get_profile(self.proxieservers), self.container) _print(self, None, f"Ny profil hämtad {self.email}") self.update_time() else: sleep(40) def accept_cookies(self): """Accepterar cookies""" self.browser.open("https://mbasic.facebook.com") soup = BeautifulSoup(str(self.browser.parsed), "lxml") if "accept all" not in soup.text.lower(): sleep_(2) cookie_accept_url = "https://mbasic.facebook.com/cookie/consent-page" self.browser.open(cookie_accept_url) sleep_(2) try: form = self.browser.get_form() self.browser.submit_form(form) _print(self, None, f"Accepterade cookies för {self.name}") sleep_(2) update_cookie(self.browser.session.cookies, self) except: try: write_error(12, self, soup=self.browser.parsed) except: pass _print(self, None, f"Accepterade inte cookies för {self.name}") def login(self): """Loggar in på Facebook.""" print("Loggar in {}".format(self.name)) # Gå till log in-sidan self.browser.open("https://mbasic.facebook.com/login") # Kolla om browser redan är inloggad soup = BeautifulSoup(str(self.browser.parsed), "lxml") if "log out" in soup.text.lower(): print("Redan inloggad.") try: # Hitta och fyll i formulär form = self.browser.get_form(id="login_form") form["email"].value = self.email form["pass"].value = self.pwd self.browser.submit_form(form, submit=form["login"]) # Vänta lite och uppdatera cookie print("Loggade in.") sleep_(2) self.open(url_bas) sleep_(2) except TypeError: try: write_error(11, self, soup=soup, profile=self.name) except: pass def update_cookie(self, cookie): self.cookie = cookie db.update_document({"_id": self.doc["_id"], "cookie": cookie}, check_rev=False) class Proxies: def __init__(self): self.proxies = [ "gb25-wg.socks5.mullvad.net:1080", "gb26-wg.socks5.mullvad.net:1080", "gb27-wg.socks5.mullvad.net:1080", "gb28-wg.socks5.mullvad.net:1080", "gb29-wg.socks5.mullvad.net:1080", ] def get_proxie(self): return self.proxies.pop(random.randrange(0, len(self.proxies), 1)) class Friend: def __init__(self, user): self.collection = "members" self.user = user # The friends friend self.username = "" self.url = "" self.name = "" def add_to_db(self): db.insert_document( self.collection, { "_key": str(self.username), "url": url_bas + self.url, "name": self.name, }, overwrite_mode="update", silent=True, ) class Reaction: def __init__(self, user, friend_username, picture_id): self.collection = "picture_reactions" self.user = user self.picture_id = picture_id self.user_name_friend = friend_username self.type = False def get_dict(self): key = str(self.picture_id) + "_" + str(self.user_name_friend) return { "_to": "members/" + str(self.user), "_from": "members/" + str(self.user_name_friend), "_key": key, "_id": "picture_reactions/" + key, "picture": self.picture_id, "reaction": self.type, }