from getpass import getpass from random import randint from time import sleep import json from datetime import datetime from arango import ArangoClient from config import * for i in range(0, 6, 1): if i == 5: exit() try: # Om scriptet körs på Macbook finns lösenordet i en fil with open("../password_arango.txt") as f: pwd = f.readline() except FileNotFoundError: if pwd == None: pwd = getpass(f'Lösenord för {user_arango}: ') try: db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd) db.collection('members').random() # För att testa löseordet/kopplingen. break except: print("Fel lösenord.") sleep(1) from helpers import now, _print, nowstamp, sleep_ from classes import Profile def checked_members(): cursor = db.aql.execute( """ FOR doc IN members FILTER doc.checked == true RETURN doc._key """ ) members_checked = set([doc for doc in cursor]) return members_checked def update_inuse(profile): db.collection("profiles").update(profile["doc"]["id"]) def count_docs(col): cursor = db.aql.execute( """ FOR doc IN @@col COLLECT WITH COUNT INTO length RETURN length """, bind_vars={"@col": col}, ) return cursor.next() def report_blocked(profile): try: db.insert_document( "reports", { "_key": str(profile.name).replace(' ', ''), "profile": profile.__dict__ }, overwrite=True, ) except: _print(profile, profile.container, f'Kunde inte rapportera blockerad: {profile.name}.') def get_profile(db=db, proxieservers='mullvad', collection='profiles'): """ Hämtar profil från profiles """ if proxieservers != 'mullvad': collection = f'profiles_{proxieservers}' #TODO Byt namn på profiles till profiles_mullvad i DB while True: cursor = db.aql.execute( """ FOR doc IN @@col FILTER doc.in_use < @inuse RETURN doc """, bind_vars={"inuse": nowstamp() - 1200, '@col': collection} ) profiles = [profile for profile in cursor] if profiles == []: sleep(180) if proxieservers=='test': # Om det är ett test profile = profiles[0] else: profile = profiles[randint(0, len(profiles) - 1)] return profile def friends_of_user(user): """Returnernar användare som reagerat på user:s bilder""" cursor = db.aql.execute( """ FOR doc IN picture_reactions FILTER doc._to == @user RETURN DISTINCT doc._from """, bind_vars={"user": "members/" + user}, ) return [doc[8:] for doc in cursor] def remove_profile(profile): """ Tar bort en blockerad profil från databasen. """ _print(profile, None, f"Tar bort {profile.name}.") db.collection("profiles").delete( profile.doc["_key"], silent=True, ignore_missing=True ) _print(profile, profile.container, f"{profile.name} blockerad och borttagen {now()}.") # TODO #2 Bättre funktion för backup av databasen def arango_connect(pwd): return ArangoClient(hosts=host_arango).db( db_arango, username=user_arango, password=pwd ) def check_for_user(username, mode=''): """ Checks if a user exist in db and if it's checked """ checked = False if db.collection("members").has(username): member = db.collection('members').get(username) if 'checked' in member: if member['checked'] == True: checked = True if mode == 'all': if 'mode' in member: if member['mode'] in ['few', 'solo']: checked = False return checked def check_for_picture(id): """ Checks if a picture exist in db """ return db.collection("pictures").has(id) def get_user(collection="lookups"): """ Hämtar användare att kolla upp från lookups """ if collection == "leak_lookups": doc = db.collection("leak_lookups").random() doc["other"] = [] db.collection(collection).delete(doc["_key"]) else: cursor = db.aql.execute( """ FOR doc IN @@col RETURN doc """, bind_vars={"@col": collection}, ) try: doc = cursor.next() if "other" not in doc: doc["other"] = [] db.collection(collection).delete(doc["_key"]) except StopIteration: doc = None return doc def backup(db): """Skapar en json-backup och statistik för specificerade collections. Args: db: databaskoppling till aktuell databas """ while True: if not datetime.now().strftime("%H") == '03' and int(datetime.now().strftime("%M")) < 10: sleep(120) continue collections = ["members", "pictures", "picture_reactions", "profiles", "stats"] for col in collections: l = [] count = 0 icount = 0 for doc in db.collection(col).all(): count += 1 l.append(doc) if count == 1000000: icount += 1 count = 0 with open(f"data/backup_{col}_{icount}.json", "w") as f: json.dump(l, f) l = [] icount += 1 with open(f"data/backup_{col}_{icount}.json", "w") as f: json.dump(l, f) l = [] print(f"Senaste backup: {now()}") write_stats() sleep(82800) def write_stats(continuous=False): while True: d = {} for col in db.collections(): if not col['system']: d[col['name']] = db.collection(col['name']).count() del d['stats'] #d['time'] = now() cursor = db.aql.execute( """ FOR doc IN members FILTER doc.checked == true COLLECT WITH COUNT INTO length RETURN length """ ) d['checked_members'] = cursor.next() # Hur många konton per säljare som finns kvar cursor = db.aql.execute( ''' for doc in profiles filter has(doc, "vendor") COLLECT vendor = doc.vendor WITH COUNT INTO length RETURN { "vendor" : vendor, "active" : length } ''') d['active_vendors'] = [doc for doc in cursor] d['_key'] = now()[:13] db.insert_document( "stats", d, overwrite=True) if continuous: sleep(86400) else: break def blocked_profile(profile, proxieservers): """ Tar bort profilen som blivit blockad och returnerar en ny. """ _print(profile, None, f'Rapporterar att {profile.name} blockats.') report_blocked(profile) _print(profile, None, f'Tar bort {profile.name} från databasen.') remove_profile(profile) _print(profile, None, f'Hämtar en ny profil.') profile = new_profile(profile.container, proxieservers) return profile def new_profile(container, proxieservers): """ Hämtar en ny profil. """ profile = Profile(get_profile(proxieservers=proxieservers), container, proxieservers) _print(profile, None, f'Hämtade profilen {profile.name}. Login = {profile.logged_in}.') if profile.logged_in == False: profile.accept_cookies() sleep_(2) profile.login() sleep_(2) try: profile.open(url_bas) if "accept all" in profile.viewing().text.lower(): _print(profile, None, f'Accepterar cookies {profile.name}.') profile.accept_cookies() sleep_(3) except: pass return profile def find_id(): "https://mbasic.facebook.com/leif.jonsson.98499/about?lst=100064897389168%3A100000134933241%3A1615858816" cursor = db.aql.execute( """ for doc in members filter has(doc, "about") filter doc.facebook_id == '' filter doc.about != false return doc """, ) n = 0 for doc in cursor: about = doc['about'] try: doc['facebook_id'] = about[about.find('%')+3: about.rfind('%')] db.update_document(doc, silent=True, check_rev=False) #sleep(0.01) n += 1 print(n, end = '\r') except AttributeError: pass db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)