fb-scraper/facebook/classes.py

import pickle
import random
from datetime import datetime

import requests
import werkzeug
from bs4 import BeautifulSoup

werkzeug.cached_property = werkzeug.utils.cached_property
from robobrowser import RoboBrowser

from arangodb import db
from config import *
from helpers import sleep_, update_cookie


class User:
    def __init__(self, username):
        self.collection = "members"
        self.username = str(username)
        self.fetched = datetime.now().strftime("%Y%m%d_%H:%M:%S")
        self.url_coverphotos = ''
        self.id = ''
        self.url_likes = ''
        self.url_about = ''
        self.url_timeline = ''
        self.profile_pictures = ''
        self.url = ''
        self.name = ''
        self.url_other_picture = ''
        self.doc

    def add_to_db(self):
        # Lägg till profilen till arrango
        db.insert_document(
            self.collection,
            {
                "_key": self.username,
                "url": self.url,
                "name": self.name,
                "profile_pictures": self.profile_pictures,
                "facebook_id": self.id,
                "timeline": self.url_timeline,
                "likes": self.url_likes,
                "about": self.url_about,
                "cover photos": self.url_coverphotos,
                "fetched": self.fetched
            },
            overwrite_mode="update",
            silent=True, 
            keep_none=False
        )

    def checked(self):
        db.update_document(
            {
            "_id": "members/" + str(self.username),
            "checked": True,
            "pictures_checked": self.profile_pictures, 
            })


class Picture:
    def __init__(self, user):
        self.collection = "pictures"
        self.user = user
        self.id = ''
        self.url_full = ''
        self.date = ''
        self.url = ''
        self.no_reactions = ''
        self.reactions = []

    def add_to_db(self):
        db.insert_document(
            self.collection,
            {
                "_key": self.id,
                "url": self.url_full,
                "date": self.date,
                "url": self.url,
                "no_reactions": self.no_reactions,
                "user": self.user,
            },
            overwrite_mode="update",
            silent=True, 
            keep_none=False
        )

class Profile:
    def __init__(self, profile):   
        # Uppdatera dokumentet i arango
        self.doc = profile
        self.doc['in_use'] = True
        db.update_document(self.doc, check_rev=False)

        # Användaruppgifter
        self.name = self.doc["name"].strip()
        self.email = self.doc["email"]
        self.pwd = self.doc["pwd"]
        self.server = self.doc["server"]
        
        self.blocked = False

        # Ange proxies
        session = requests.Session()
        session.proxies = {
            "https": "socks5://'8155249667566524'@{}".format(self.server),
            "http": "socks5://'8155249667566524'@{}".format(self.server),
        }

        # Starta browser
        user_agent = "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e Safari/602.1"
        self.browser = RoboBrowser(
            session=session, user_agent=user_agent, history=False, parser="lxml"
        )
        try:
            self.browser.session.cookies = pickle.load(
                open("data/cookie_{}.pkl".format(self.name), "rb")
            )
            self.logged_in = True
        except:
            self.logged_in = False

    def viewing(self):
        """ Returnerar browser i html-format """
        return self.browser.parsed
        

    def accept_cookies(self):
        """ Accepterar cookies """
        self.browser.open("https://mbasic.facebook.com")
        soup = BeautifulSoup(str(self.browser.parsed), "lxml")
        if 'accept all' not in soup.text.lower():
            sleep_(2)
            cookie_accept_url = "https://mbasic.facebook.com/cookie/consent-page" 
            self.browser.open(cookie_accept_url)
        sleep_(2)
        try:
            form = self.browser.get_form()
            self.browser.submit_form(form)
            print(f"Accepterade cookies för {self.name}")
            sleep_(2)
            update_cookie(self.browser.session.cookies, self.name)
        except Exception as e:
            print(f"Accepterade inte cookies för {self.name}")
    
    def login(self):
        """ Loggar in på Facebook. """

        print("Loggar in {}".format(self.name))

        # Gå till log in-sidan
        self.browser.open("https://mbasic.facebook.com/login")

        # Kolla om browser redan är inloggad
        soup = BeautifulSoup(str(self.browser.parsed), "lxml")
        if 'log out' in soup.text.lower():
            print("Redan inloggad.")
         
        # Hitta och fyll i formulär
        form = self.browser.get_form(id="login_form")
        form["email"].value = self.email
        form["pass"].value = self.pwd
        self.browser.submit_form(form, submit=form["login"])
        # Vänta lite och uppdatera cookie
        print("Loggade in.") 
        sleep_(2)

    def unused(self):
        """ Sätter user till False för valda profiler """
        self.doc["in_use"] = False
        db.update_document(self.doc['_key'], silent=True)

class Proxies:
    def __init__(self):
        self.proxies = [
                'gb25-wg.socks5.mullvad.net:1080',
                'gb26-wg.socks5.mullvad.net:1080',
                'gb27-wg.socks5.mullvad.net:1080',
                'gb28-wg.socks5.mullvad.net:1080',
                'gb29-wg.socks5.mullvad.net:1080'
                ]
    def get_proxie(self):
        return self.proxies.pop(random.randrange(0, len(self.proxies), 1))

class Friend:
    def __init__(self, user):
        self.collection = "members"
        self.user = user  # The friends friend
        self.username = ''
        self.url = ''
        self.name = ''
        self.single = ''

    def add_to_db(self):
        db.insert_document(
            self.collection,
            {
                "_key": self.username,
                "url": url_bas + self.url,
                "name": self.name,
            },
            overwrite_mode="update",
            silent=True,
        )


class Reaction:
    def __init__(self, user, friend_username, picture_id):
        self.collection = "picture_reactions"
        self.user = user
        self.picture_id = picture_id
        self.user_name_friend = friend_username
        self.type = False

    def get_dict(self):
        key = str(self.picture_id) + "_" + str(self.user_name_friend)
        return {
            "_to": "members/" + str(self.user),
            "_from": "members/" + str(self.user_name_friend),
            "_key": key,
            "_id": "picture_reactions/" + key,
            "picture": self.picture_id,
            "reaction": self.type,
        }