fb-scraper/facebook/arangodb.py

from getpass import getpass
from random import randint
from time import sleep
import json
from datetime import datetime
from json2html import json2html

from arango import ArangoClient

from config import *


for i in range(0, 6, 1):
    if i == 5:
        exit()
    try:
        # Om scriptet körs på Macbook finns lösenordet i en fil
        with open("../password_arango.txt") as f:
            pwd = f.readline()
    except FileNotFoundError:
        if 'pwd' not in globals():
            pwd = getpass(f'Lösenord för {user_arango}: ')

    try:
        db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)
        db.collection('members').random() # För att testa löseordet/kopplingen.
        break
    except:
        print("Fel lösenord.")
        sleep(1)

from helpers import now, _print, nowstamp, sleep_
from classes import Profile


def checked_members():
    cursor = db.aql.execute(
        """
            FOR doc IN members
            FILTER doc.checked == true
            RETURN doc._key
            """
    )
    members_checked = set([doc for doc in cursor])
    return members_checked


def update_inuse(profile):
    db.collection("profiles").update(profile["doc"]["id"])


def count_docs(col):
    cursor = db.aql.execute(
        """
            FOR doc IN @@col
            COLLECT WITH COUNT INTO length
            RETURN length
            """,
        bind_vars={"@col": col},
    )
    return cursor.next()


def report_blocked(profile):
    try:
        db.insert_document(
            "reports",
            {
                "_key": str(profile.name).replace(' ', ''),
                "profile": profile.__dict__
            },
            overwrite=True,
        )
    except:
        _print(profile, profile.container, f'Kunde inte rapportera blockerad: {profile.name}.')


def get_profile(db=db, collection='mullvad'):
    """ Hämtar profil från profiles """


    while True:
        cursor = db.aql.execute(
            """
                FOR doc IN @@col
                FILTER doc.in_use < @inuse
                RETURN doc
                """,
            bind_vars={"inuse": nowstamp() - 1200, '@col': f'profiles_{collection}'}
        )

        profiles = [profile for profile in cursor]
        if profiles == []:
            sleep(180)
        else:
            profile = profiles[randint(0, len(profiles) - 1)]
        return profile
            

def friends_of_user(user):
    """Returnernar användare som reagerat på user:s bilder"""
    cursor = db.aql.execute(
        """
        FOR doc IN picture_reactions
        FILTER doc._to == @user
        RETURN DISTINCT doc._from
        """,
        bind_vars={"user": "members/" + user},
    )
    return [doc[8:] for doc in cursor]


def remove_profile(profile, proxieservers='mullvad'):
    """ Tar bort en blockerad profil från databasen. """
    _print(profile, None, f"Tar bort {profile.name}.")

    db.collection(f'profiles_{proxieservers}').delete(
        profile.doc["_key"], silent=True, ignore_missing=True
    )
    _print(profile, profile.container, f"{profile.name} blockerad och borttagen {now()}.")


# TODO #2 Bättre funktion för backup av databasen

def arango_connect(pwd):
    return ArangoClient(hosts=host_arango).db(
        db_arango, username=user_arango, password=pwd
    )


def check_for_user(username, mode=''):
    """ Checks if a user exist in db and if it's checked """
    checked = False
    if db.collection("members").has(username):
        member = db.collection('members').get(username)
        if 'checked' in member:
            if member['checked'] == True:
                checked = True
        if mode == 'all':
            if 'mode' in member:
                if member['mode'] in ['few', 'solo']:
                    checked = False
    return checked
        

def check_for_picture(id):
    """ Checks if a picture exist in db """
    return db.collection("pictures").has(id)


def get_user(collection="lookups"):
    """ Hämtar användare att kolla upp från lookups """

    if collection == "leak_lookups":
        doc = db.collection("leak_lookups").random()
        doc["other"] = []
        db.collection(collection).delete(doc["_key"])
    else:
        cursor = db.aql.execute(
            """
                FOR doc IN @@col
                RETURN doc
                """,
            bind_vars={"@col": collection},
        )
        try:
            doc = cursor.next()
            if "other" not in doc:
                doc["other"] = []
            db.collection(collection).delete(doc["_key"])
        except StopIteration:
            doc = None

    return doc


def backup(db):
    """Skapar en json-backup och statistik för specificerade collections.

    Args:
        db: databaskoppling till aktuell databas
    """
    while True:
        if not datetime.now().strftime("%H") == '03' and int(datetime.now().strftime("%M")) < 10:
            sleep(120)
            continue
        collections = ["members", "pictures", "picture_reactions", "profiles", "stats"]
        for col in collections:
            l = []
            count = 0
            icount = 0
            for doc in db.collection(col).all():
                count += 1
                l.append(doc)
                if count == 1000000:
                    icount += 1
                    count = 0
                    with open(f"data/backup_{col}_{icount}.json", "w") as f:
                        json.dump(l, f)
                        l = []
            icount += 1
            with open(f"data/backup_{col}_{icount}.json", "w") as f:   
                json.dump(l, f)
                l = []
        print(f"Senaste backup: {now()}")
        write_stats()
        sleep(82800)
        

def write_stats(continuous=False):
    while True:
        d = {}
        for col in db.collections():
            if not col['system']:
                d[col['name']] = db.collection(col['name']).count()
        del d['stats']
        #d['time'] = now()
        cursor = db.aql.execute(
            """
            FOR doc IN members
            FILTER doc.checked == true
            COLLECT WITH COUNT INTO length
            RETURN length
                    """
                )
        d['checked_members'] = cursor.next()


        # Hur många konton per säljare som finns kvar
        cursor = db.aql.execute(
        '''
        for doc in profiles
        filter has(doc, "vendor")
        COLLECT vendor = doc.vendor WITH COUNT INTO length
        RETURN { 
        "vendor" : vendor, 
        "active" : length 
        }
        ''')
        d['active_vendors'] = [doc for doc in cursor]

        d['_key'] = now()[:13]
        db.insert_document( "stats", d, overwrite=True)

        # Skriv en html-fil
        with open('webbapp/templates/stats.html', 'a+') as html:
            html.truncate(0)
            html.write('<!DOCTYPE html> <br>')

            html.write(json2html.convert(json = d))
        
        # Sov för att fortsätta senare
        if continuous:
            sleep(86400)
        else:
            break

def blocked_profile(profile, proxieservers):
    """ Tar bort profilen som blivit blockad och returnerar en ny. """
    _print(profile, None, f'Rapporterar att {profile.name} blockats.')
    report_blocked(profile)
    _print(profile, None, f'Tar bort {profile.name} från databasen.')
    remove_profile(profile, proxieservers)
    _print(profile, None, f'Hämtar en ny profil.')
    profile = new_profile(profile.container, proxieservers)
    return profile


def new_profile(container, proxieservers):
    """ Hämtar en ny profil. """
    profile = Profile(get_profile(proxieservers=proxieservers), container, proxieservers)
    _print(profile, None, f'Hämtade profilen {profile.name}. Login = {profile.logged_in}.')
    if profile.logged_in == False:
        profile.accept_cookies()
        sleep_(2)
        profile.login()
    sleep_(2)
    try:
        profile.open(url_bas)
        if "accept all" in profile.viewing().text.lower():
            _print(profile, None, f'Accepterar cookies {profile.name}.')
            profile.accept_cookies()
            sleep_(3)
    except:
        pass
    return profile

def find_id():
    "https://mbasic.facebook.com/leif.jonsson.98499/about?lst=100064897389168%3A100000134933241%3A1615858816"

    cursor = db.aql.execute(
        """
        for doc in members
        filter has(doc, "about")
        filter doc.facebook_id == ''
        filter doc.about != false
        return doc
        """,
    )

    n = 0
    for doc in cursor:
        about = doc['about']
        try:
            doc['facebook_id'] = about[about.find('%')+3: about.rfind('%')]
            db.update_document(doc, silent=True, check_rev=False)
            #sleep(0.01)
            n += 1
            print(n, end = '\r')
        except AttributeError:
            pass
        
        

db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)