lasse
/
fb-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

No profile list, no all_checked

Browse Source
pull/5/head
Lasse Edfast 5 years ago
parent fcb0654671
commit 654dab46eb
1 changed files with 135 additions and 195 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 330
      facebook/__main__.py

330
facebook/__main__.py
Unescape View File

@ -1,7 +1,7 @@
import random
import traceback
from getopt import GetoptError, getopt
from sys import argv, exit
from sys import argv
from time import sleep
from subprocess import check_output
from re import split
@ -13,64 +13,40 @@ from helpers import sleep_, write_error, _print
from scrapers import profile_picture_reactions
def finish(profile, profiles, users):
""" Avslutar: skriver rapport och gör profilerna oanvända """
for profile in profiles:
profile.unused()
try:
e = traceback.format_exc()
except:
e = 'Unknown traceback.'
_print(profile, None, e.split('\n'))
write_report(users)
exit()
def blocked_profile(profile):
""" Tar bort profilen som blivit blockad och returnerar en ny. """
report_blocked(profile)
remove_profile(profile)
return new_profile()
def blocked_profile(profile, profiles):
""" Tar bort profilen från listan med profiles, tar bort ur databasen och returnerar uppdaterad lista. """
report_blocked(profile, users)
remove_profile(profile)
# Ta bort från listan på fb-profiler som används
profiles.remove(profile)
# Försök lägga till en ny fb-profil (om det finns en skapad och ledig i databasen)
try:
doc = get_profile()
profiles.append(Profile(doc, lookingup))
_print(profile, None, f"Laddat ny profil: {profiles[-1].name}")
sleep(3)
except:
_print(profile, None, "Det behövs nya profiler...")
if len(profiles) == 0:
finish(profile, profiles, users)
for s in range(0, int(1600 / len(profiles))):
print(user, f"Sover {600-s} sekunder till... ", end="\r")
_print(profile, None, 'Långsover...')
for s in range (3600, 0, -1):
print(f'Sover {s} sekunder till...', end='\r')
sleep(1)
return profiles
def new_profile():
""" Hämtar en ny profil. """
profile = Profile(get_profile(), container)
if profile.logged_in == False:
profile.accept_cookies()
sleep_(2)
profile.login()
return profile
if __name__ == "__main__":
print()
if gethostname() not in ['macbook.local']: # Lägg till för studiodatorn
# Hämta namn för containern där skriptet körs
# Hämta namn för containern där skriptet körs
if gethostname() not in ['macbook.local']: # TODO Lägg till för studiodatorn
try:
lookingups = check_output(['docker', 'lookingup', 'ls']).decode()
lookingup = split('\W\W+', lookingups.split('\n')[1])[-1]
containers = check_output(['docker', 'lookingup', 'ls']).decode()
container = split('\W\W+', containers.split('\n')[1])[-1]
except FileNotFoundError:
pass
else:
lookingup_name = 'macbook'
container = 'macbook'
# Argument och alternativ
argv = argv[1:]
try:
opts, args = getopt(argv, "bm:u:o:", ['backup=',"mode=", "users=", "other="])
opts, args = getopt(argv, "bm:u:o:", ['backup=',"mode=", "user=", "other="])
for o, a in opts:
# mode_nr används för hur ofta profile ska roteras
if o in ["-m", "--mode"]:
@ -88,10 +64,13 @@ if __name__ == "__main__":
for o, a in opts:
if o in ["-u", "--user"]:
try:
users = [
User(str(i).strip(), mode)
for i in [(str(i).strip()) for i in a.split(",")]
]
if a.strip()== 'leak':
lookups = 'leak_lookups'
user = get_user(collection='leak_lookups')['_key']
else:
lookups = 'lookups'
user = a
user = User(str(user).strip(), mode)
except StopIteration:
raise Exception
if o in ["-o", "--other"]:
@ -101,18 +80,13 @@ if __name__ == "__main__":
backup(db)
sleep(21600)
if "user" not in globals():
lookups = 'lookups'
user = User(str(input("Vem/vilka vill du kolla bilder för? ")).strip(), mode)
if "users" not in globals():
users = [
User(str(i).strip(), mode)
for i in input("Vem/vilka vill du kolla bilder för? ").split(",")
]
except GetoptError:
users = [
User(str(i).strip(), mode)
for i in input("Vem/vilka vill du kolla bilder för? ").split(",")
]
lookups = 'lookups'
user = User(str(input("Vem/vilka vill du kolla bilder för? ")).strip(), mode)
mode = input("Söka efter alla, första/sida eller första? (all, few, single)? ").lower().strip()
if mode == '':
@ -122,153 +96,119 @@ if __name__ == "__main__":
l = []
for url in url_other_pictures:
l.append(url[url.find('facebook.com') + 12:])
users[0].url_other_pictures = l
user.url_other_pictures = l
print("Kollar profilbilder för:")
for user in users:
print("-", user.username)
print()
if 'lookingup' not in globals():
usernames = [user.username for user in users]
if len(usernames) == 1:
lookingup = usernames[0]
else:
lookingup = '-'.join(usernames)
# Skapa tre olika profiler att besöka Facebook med
profiles = []
for i in range(0, 3):
doc = get_profile()
profile = Profile(doc, lookingup)
profile.browser.open("https://api.ipify.org")
print(f"Profil {profile.name} använder IP-adress {profile.viewing().text}."
)
if profile.logged_in == False:
profile.accept_cookies()
sleep_(2)
profile.login()
profiles.append(profile)
print()
sleep(3)
profile_nr = 0
profile = profiles[profile_nr]
_print(profile, user, f"Börjar med profilen {profile.name}")
# Hämta profil
profile = new_profile()
sleep(3)
# Gå igenom de användare som efterfrågats
#try:
while True:
for user in users:
profile.users_checked += 1
# Set för kollade bilder och kollade medlemmar
all_pictures = set([doc["_key"] for doc in db.collection("pictures").all()])
all_pictures_start = all_pictures.copy()
members_checked = checked_members()
for p in profiles:
p.lookingup = user.username
# Hämta reaktioner för den första användaren
if any([user.username not in members_checked, mode == 'force']):
try:
t = 0
while t < 5:
t += 1
profile = profile_picture_reactions(profile, user, all_pictures, first_user=True, mode=mode)
if profile.blocked == True:
profiles = blocked_profile(profile, profiles)
else:
break
except:
_print(profile, user, traceback.format_exc())
if len(users) == 1:
for p in profiles:
p.unused()
friends = friends_of_user(user.username)
friends_unchecked = list(set(friends) - set(members_checked))
_print(profile, user, f"\nKlar med, {user.username}\n")
_print(profile, user, f"Vänner som reagerat: {len(friends)}")
_print(profile, user, "\nVänner att kolla:")
for friend in friends_unchecked:
print(friend)
_print(profile, user, [friend for friend in friends_unchecked], silent=True)
print()
if lookups == 'leak_lookups':
profile.browser.open(url_bas + "/" + user.username)
url = profile.browser.state.url.strip('/').strip('?_rdr')
user = User(str(url[url.rfind('/') + 1:]).strip(), mode)
sleep_(4)
# Hämta reaktioner för users vänner (som reagerat)
count_friends = 0
for friend in friends_unchecked:
profile.users_checked += 1
if profile.users_checked > 30:
_print(profile, None, 'Långsover...')
for s in range(2600, 0, -1):
print(f'Sover {s} sekunder till...', end='\r')
sleep(1)
for p in profiles:
_print(p, None, f'Långsovit. {p.name} hade kollat {p.users_checked} användare.')
p.users_checked = 0
count_friends += 1
user = User(str(friend), mode, other_pictures=[])
sleep_(2)
try:
p = profile_picture_reactions(
profile, user, all_pictures, mode=mode
)
if isinstance(p, Profile):
profile = p
print(f"Kollar profilbilder för {user.username}")
except Exception as e: # Fel4
write_error(
4,
e=e,
user=user,
profile=profile,
traceback=traceback.format_exc(),
soup=profile.viewing(),
)
_print(profile, user, f"\nFel: {str(user.username)}\n")
sleep_(15)
if profile.blocked == False:
_print(profile, user, f"Klar med {user.username} \n")
if 'container' not in globals:
container = str(user.username)
profile.container = container
# Rotera fb-profiler
if count_friends > 5 * mode_nr:
if random.randrange(0, 2, 1) == 1:
profile_nr += 1
if profile_nr >= len(profiles):
profile_nr = 0
count_friends = 0
_print(profile, user, f"Växlar till {profiles[profile_nr].name}")
elif count_friends > 9 * mode_nr:
profile_nr += 1
if profile_nr >= len(profiles):
profile_nr = 0
count_friends = 0
_print(profile, user, f"Växlar till {profiles[profile_nr].name}")
profile = profiles[profile_nr]
_print(profile, user, f"Börjar med profilen {profile.name}")
profile.users_checked += 1
# Set för kollade bilder och kollade medlemmar
#all_pictures = set([doc["_key"] for doc in db.collection("pictures").all()])
#all_pictures_start = all_pictures.copy()
#members_checked = checked_members()
elif profile.blocked == True:
_print(profile, user, f"Tar bort {profile.name}\n".upper(), sleeptime=1)
profiles = blocked_profile(profile, profiles)
profile_nr =len(profiles) -1
profile = profiles[profile_nr]
_print(profile, None, [p.name for p in profiles])
# Hämta ny användare från databasen (lookups) när alla är genomgångna
doc_new_user = get_user()
if doc_new_user == None:
_print(profile, user, 'No more to look up.')
finish(profile, profiles, users)
else:
new_user = User(doc_new_user['_key'], mode, doc_new_user['other'])
for p in profiles:
p.lookingup = new_user.username
users.append(new_user)
# Hämta reaktioner för den första användaren
if any([not check_for_user(user.username), mode == 'force']):
try:
while True:
# Uppdatera in_use
profile.update_time()
profile = profile_picture_reactions(profile, user, first_user=True, mode=mode)
if profile.blocked:
profile = blocked_profile(profile)
else:
break
except:
_print(profile, user, traceback.format_exc())
friends = friends_of_user(user.username)
_print(profile, user, f"\nKlar med, {user.username}\n")
_print(profile, user, f"Vänner som reagerat: {len(friends)}")
_print(profile, user, "\nVänner att kolla:")
friends_unchecked = []
for friend in friends:
if not check_for_user(friend):
print(friend)
friends_unchecked.append(friend)
_print(profile, user, [friends_unchecked], silent=True)
print()
# Hämta reaktioner för users vänner (som reagerat)
count_friends = 0
for friend in friends_unchecked:
count_friends += 1
user = User(str(friend), mode, other_pictures=[])
sleep_(2)
# Uppdatera in_use
profile.update_time()
try:
p = profile_picture_reactions(
profile, user, mode=mode
)
if isinstance(p, Profile):
profile = p
except Exception as e: # Fel4
write_error(
4,
profile,
e=e,
user=user,
traceback=traceback.format_exc(),
soup=profile.viewing(),
)
_print(profile, user, f"\nFel: {str(user.username)}\n")
sleep_(15)
if not profile.blocked:
_print(profile, user, f"Klar med {user.username} \n")
# Rotera fb-profiler
if count_friends > 2 * mode_nr:
if random.randrange(0, 2, 1) == 1:
profile = new_profile()
count_friends = 0
_print(profile, user, f"Växlar till {profile.name}")
elif count_friends > 4 * mode_nr:
profile = new_profile()
count_friends = 0
_print(profile, user, f"Växlar till {profile.name}")
elif profile.blocked:
_print(profile, user, f"Tar bort {profile.name}\n".upper(), sleeptime=1)
profile = blocked_profile(profile)
_print(profile, user, f"Växlar till {profile.name}")
# except:
# finish(profile, profiles, users)
# Hämta ny användare från databasen när alla är genomgångna
while True:
user = get_user(collection=lookups)
if user == None:
sleep(300)
else:
user = User(str(user['_key']).strip(), mode)
break

Write Preview
Loading…
Cancel
Save