lasse
/
fb-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Divided into modules

Browse Source
pull/5/head
Lasse Edfast 5 years ago
parent 5e60067d6b
commit 2ee1d93d48
10 changed files with 1655 additions and 792 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      .gitignore
  2. 10
      Dockerfile
  3. 173
      facebook/__main__.py
  4. 76
      facebook/arangodb.py
  5. 225
      facebook/classes.py
  6. 9
      facebook/config.py
  7. 70
      facebook/helpers.py
  8. 787
      facebook_reactions.py
  9. 824
      htmlerror.html
  10. 267
      scrapers.py

4
.gitignore vendored
Unescape View File

@ -2,3 +2,7 @@
/.venv
/.vscode
/__pycache__
*.json
*.pkl
/facebook/test.py
/data/*

10
Dockerfile
Unescape View File

@ -7,15 +7,17 @@ COPY requirements.txt .
RUN pip install -r requirements.txt
#RUN apt-get install build-essential libssl-dev libffi-dev python-dev
ADD data /data
COPY facebook_reactions.py .
COPY main.py .
CMD [ "python", "./facebook_reactions.py" ]
ENTRYPOINT [ "python", "./main.py" ]
CMD ["",""]
# BUILD:
# docker buildx create --use
#docker buildx build --platform linux/arm -t l3224/fb-reactions:pi --push .
# START
# docker run -it --name fb1 -v vol1:/data l3224/fb-reactions:latest
# docker run -it --name fb1 -v vol1:/data l3224/fb-reactions:latest [-s -u user1,user2]

173
facebook/__main__.py
Unescape View File

@ -0,0 +1,173 @@
import os
import random
import traceback
from datetime import datetime
from getopt import GetoptError, getopt
from sys import argv
from time import sleep
import arangodb
from arangodb import db
from classes import Profile, User
from helpers import sleep_, write_error
from scrapers import profile_picture_reactions
# import werkzeug
# werkzeug.cached_property = werkzeug.utils.cached_property
# from arango import ArangoClient
if __name__ == "__main__":
print()
# Säkerställ att arbetsmappen är samma som den där scriptet ligger
os.chdir(os.path.dirname(__file__))
# Argument och alternativ
argv = argv[1:]
try:
opts, args = getopt(argv, "su:o:", ["single", "users=", "other="])
single = True if "-s" in [o[0] for o in opts] else False
for o, a in opts:
if o in ["-u", "--user"]:
users = [
User(str(i).strip())
for i in [(str(i).strip()) for i in a.split(",")]
]
if o in ["-o", "--other"]:
url_other_picture = a
if "users" not in globals():
users = [
User(str(i).strip())
for i in input("Vem/vilka vill du kolla bilder för? ").split(",")
]
except GetoptError:
users = [
User(str(i).strip())
for i in input("Vem/vilka vill du kolla bilder för? ").split(",")
]
single = (
True
if input("Söka bara en bild (single)?").lower() in ["ja, yes, j, y"]
else False
)
if "url_other_picture" in globals():
users[0].url_other_picture = url_other_picture[url_other_picture.find('facebook.com') + 12:]
print("Kollar profilbilder för:")
for user in users:
print("-", user.username)
print()
# Skapa tre olika profiler att besöka Facebook med
profiles = []
for i in range(0, 3):
doc = arangodb.get_profile()
profile = Profile(doc)
profile.browser.open("https://api.ipify.org")
print(
f"Profil {profile.name} använder IP-adress {profile.viewing().text}."
)
if profile.logged_in == False:
profile.accept_cookies()
sleep_(2)
profile.login()
profiles.append(profile)
print()
sleep(3)
profile_nr = 1
profile = profiles[profile_nr]
print("Börjar med profilen", profile.name)
# Gå igenom de användare som efterfrågats
while True:
for user in users:
# Set för kollade bilder och kollade medlemmar
all_pictures = set([doc["_key"] for doc in db.collection("pictures").all()])
members_checked = arangodb.checked_members()
if user.username not in members_checked:# Hämta reaktioner för den första användaren LÄGG TILL NOT IN MEMBERS_CHECKED
profile_picture_reactions(profile, user, all_pictures, first=True, single=single)
friends = arangodb.friends_of_user(user.username)
friends_unchecked = list(set(friends) - set(members_checked))
# Här följer cookien med så att vi fortfarnade är inloggade
print("\nKlar med", user.username, "\n")
print("Vänner som reagerat:", len(friends))
print("Vänner att kolla:")
for friend in friends_unchecked:
print(friend)
print()
# Hämta reaktioner för den första användarens vänner (som reagerat)
count_friends = 0
for friend in friends_unchecked:
count_friends += 1
user = User(str(friend))
sleep_(2)
try:
profile_picture_reactions(
profile, user, members_checked, all_pictures
)
if profile.blocked == True:
# Ta bort profilen ur databasen
arangodb.remove_profile(profile.doc["_key"])
# Ta bort från listan på fb-profiler som används
profiles.remove(profile)
# Försök lägga till en ny fb-profil (om det finns en skapad och ledig i databasen)
try:
profiles[profile_nr] = Profile(new=True)
print("Laddat ny profil:", profiles[profile_nr].name)
sleep(3)
except e:
print("Det behövs nya profiler...")
for s in range(0, 1600 / len(profiles)):
print(f"Sover {600-s} sekunder till... ", end="\r")
profile_nr += 1
print(f"Försöker med {profiles[profile_nr].name}.")
else:
print("Klar med", user.username, "\n")
# Rotera fb-profiler
if count_friends == 6:
if random.randrange(0, 2, 1) == 1:
profile_nr += 1
count_friends = 0
print("Växlar till", profiles[profile_nr].name)
elif count_friends == 10:
profile_nr += 1
count_friends = 0
print("Växlar till", profiles[profile_nr].name)
if profile_nr > len(profiles) - 1:
profile_nr = 0
profile = profiles[profile_nr]
except Exception as e: # Fel4
write_error(
4,
e=e,
user=user.username,
traceback=traceback.format_exc(),
soup=profile.viewing(),
)
print("\nFel: ", str(user.username), "\n")
sleep_(15)
pass
# Ladda in nya användare att kolla
print("\nVem vill du kolla upp?")
user_input = input(">>> ")
if user_input in ['exit', '', 'ingen']:
for profile in profiles:
profile.unused()
break
else:
users = [User(str(i).strip()) for i in user_input.split(",")]

76
facebook/arangodb.py
Unescape View File

@ -0,0 +1,76 @@
from time import sleep
from arango import ArangoClient
from getpass import getpass
from sys import argv
from config import *
from datetime import datetime
import nacl.secret
import nacl.utils
def checked_members():
cursor = db.aql.execute(
"""
FOR doc IN members
FILTER doc.checked == true
RETURN doc._key
"""
)
members_checked = set([doc for doc in cursor])
return members_checked
def get_profile():
""" Hämtar profil om det inte gjorts förut """
cursor = db.aql.execute(
"""
FOR doc IN profiles
FILTER doc.in_use == false
FILTER doc.created == true
RETURN doc
"""
)
return cursor.next()
def friends_of_user(user):
"""Returnernar användare som reagerat på user:s bilder"""
cursor = db.aql.execute(
"""
FOR doc IN picture_reactions
FILTER doc._to == @user
RETURN DISTINCT doc._from
""",
bind_vars={"user": "members/" + user},
)
return [doc[8:] for doc in cursor]
def remove_profile(profile):
db.collection("profiles").delete(profile['_key'], silent=True, ignore_missing=True)
print(
f'{profile} blockerad och borttagen {datetime.now().strftime("%Y%m%d_%H:%M:%S")}.'
)
# Starta koppling till arangodb
# Avkryptera lösen till arango
for i in range(0, 6, 1):
if i == 5:
exit()
try:
key = "sssladnnklja" + getpass()
pwd = (
nacl.secret.SecretBox(key.encode())
.decrypt(pwd_arango, encoder=nacl.encoding.HexEncoder)
.decode("utf-8")
)
break
except:
print("Fel lösenord.")
sleep(1)
db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)

225
facebook/classes.py
Unescape View File

@ -0,0 +1,225 @@
from datetime import datetime
import json
import pickle
from bs4 import BeautifulSoup
import requests
import werkzeug
import random
werkzeug.cached_property = werkzeug.utils.cached_property
from robobrowser import RoboBrowser
from arangodb import db
from helpers import sleep_, update_cookie
from config import *
class User:
def __init__(self, username):
self.collection = "members"
self.username = str(username)
self.fetched = datetime.now().strftime("%Y%m%d_%H:%M:%S")
self.url_coverphotos = ''
self.id = ''
self.url_likes = ''
self.url_about = ''
self.url_timeline = ''
self.profile_pictures = ''
self.url = ''
self.name = ''
self.url_other_picture = ''
def add_to_db(self):
# Lägg till profilen till arrango
db.insert_document(
self.collection,
{
"_key": self.username,
"url": self.url,
"name": self.name,
"profile_pictures": self.profile_pictures,
"facebook_id": self.id,
"timeline": self.url_timeline,
"likes": self.url_likes,
"about": self.url_about,
"cover photos": self.url_coverphotos,
"fetched": self.fetched
},
overwrite_mode="update",
silent=True,
keep_none=False
)
def checked(self):
db.update_document(
{
"_id": "members/" + str(self.username),
"checked": True,
"pictures_checked": self.profile_pictures,
})
class Picture:
def __init__(self, user):
self.collection = "pictures"
self.user = user
self.id = ''
self.url_full = ''
self.date = ''
self.url = ''
self.no_reactions = ''
self.reactions = []
def add_to_db(self):
db.insert_document(
self.collection,
{
"_key": self.id,
"url": self.url_full,
"date": self.date,
"url": self.url,
"no_reactions": self.no_reactions,
"user": self.user,
},
overwrite_mode="update",
silent=True,
keep_none=False
)
class Profile:
def __init__(self, profile):
# Uppdatera dokumentet i arango
self.doc = profile
self.doc['in_use'] = True
db.update_document(self.doc, check_rev=False)
# Användaruppgifter
self.name = self.doc["name"].strip()
self.email = self.doc["email"]
self.pwd = self.doc["pwd"]
self.server = self.doc["server"]
self.blocked = False
# Ange proxies
session = requests.Session()
session.proxies = {
"https": "socks5://'8155249667566524'@{}".format(self.server),
"http": "socks5://'8155249667566524'@{}".format(self.server),
}
# Starta browser
user_agent = "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e Safari/602.1"
self.browser = RoboBrowser(
session=session, user_agent=user_agent, history=False, parser="lxml"
)
try:
self.browser.session.cookies = pickle.load(
open("data/cookie_{}.pkl".format(self.name), "rb")
)
self.logged_in = True
except:
self.logged_in = False
def viewing(self):
""" Returnerar browser i html-format """
return self.browser.parsed
def accept_cookies(self):
""" Accepterar cookies """
self.browser.open("https://mbasic.facebook.com")
soup = BeautifulSoup(str(self.browser.parsed), "lxml")
if 'accept all' not in soup.text.lower():
sleep_(2)
cookie_accept_url = "https://mbasic.facebook.com/cookie/consent-page"
self.browser.open(cookie_accept_url)
sleep_(2)
try:
form = self.browser.get_form()
self.browser.submit_form(form)
print(f"Accepterade cookies för {self.name}")
sleep_(2)
update_cookie(self.browser.session.cookies, self.name)
except Exception as e:
print(f"Accepterade inte cookies för {self.name}")
def login(self):
""" Loggar in på Facebook. """
print("Loggar in {}".format(self.name))
# Gå till log in-sidan
self.browser.open("https://mbasic.facebook.com/login")
# Kolla om browser redan är inloggad
soup = BeautifulSoup(str(self.browser.parsed), "lxml")
if 'log out' in soup.text.lower():
print("Redan inloggad.")
# Hitta och fyll i formulär
form = self.browser.get_form(id="login_form")
form["email"].value = self.email
form["pass"].value = self.pwd
self.browser.submit_form(form, submit=form["login"])
# Vänta lite och uppdatera cookie
print("Loggade in.")
sleep_(2)
def unused(self):
""" Sätter user till False för valda profiler """
self.doc["in_use"] = False
db.update_document(self.doc['_key'], silent=True)
class Proxies:
def __init__(self):
self.proxies = [
'gb25-wg.socks5.mullvad.net:1080',
'gb26-wg.socks5.mullvad.net:1080',
'gb27-wg.socks5.mullvad.net:1080',
'gb28-wg.socks5.mullvad.net:1080',
'gb29-wg.socks5.mullvad.net:1080'
]
def get_proxie(self):
return self.proxies.pop(random.randrange(0, len(self.proxies), 1))
class Friend:
def __init__(self, user):
self.collection = "members"
self.user = user # The friends friend
self.username = ''
self.url = ''
self.name = ''
self.single = ''
def add_to_db(self):
db.insert_document(
self.collection,
{
"_key": self.username,
"url": url_bas + self.url,
"name": self.name,
},
overwrite_mode="update",
silent=True,
)
class Reaction:
def __init__(self, user, friend_username, picture_id):
self.collection = "picture_reactions"
self.user = user
self.picture_id = picture_id
self.user_name_friend = friend_username
self.type = False
def get_dict(self):
key = str(self.picture_id) + "_" + str(self.user_name_friend)
return {
"_to": "members/" + str(self.user),
"_from": "members/" + str(self.user_name_friend),
"_key": key,
"_id": "picture_reactions/" + key,
"picture": self.picture_id,
"reaction": self.type,
}

9
facebook/config.py
Unescape View File

@ -0,0 +1,9 @@
# Info för arangodb
user_arango = "Lasse"
pwd_arango = "4c071768bedc259288361c07aafd8535fca546086fada4e7b5de4e2bb26b0e70fa8d348c998b90d032a5b8f3fdbae1881b843021e3475198e6fb45f58d8dc450bd52f77d"
db_arango = "facebook"
host_arango = "http://arango.lasseedfast.se"
# Andra uppgifter
url_bas = "https://mbasic.facebook.com"

70
facebook/helpers.py
Unescape View File

@ -0,0 +1,70 @@
from time import sleep
import random
import pickle
from datetime import datetime
from arangodb import db
def sleep_(t):
"""
Sover en tid nära den angivna (för att inte sökningarna ska bli för lika varandra)
"""
variation = 4 # Testa olika sovlängder för att inte få användaren blockerad
sleep(t * variation * random.randrange(85, 115, 1) / 100)
if random.randrange(0, 60, 1) == 1:
for s in range(0, 300):
print(f"Sover {300 - s} sekunder till... ", end="\r")
sleep(1)
print()
sleep(random.randrange(0, 10, 1) / 4)
def update_cookie(cookies, profile_name):
""" Uppdaterar cookie för browser """
with open("./data/cookie_{}.pkl".format(profile_name), "wb") as f:
pickle.dump(cookies, f)
def write_error(nr, e="", traceback="", soup="", user="", url="", url_name=""):
"""Skriver info efter error till arango
Args:
nr ([type]): error number
e (str, optional): error. Defaults to "".
traceback (str, optional): The traceback from traceback.format_exc(). Defaults to "".
soup (str, optional): Soup. Defaults to "".
user (str, optional): The user. Defaults to "".
url (str, optional): Url, if any. Defaults to "".
count (int, optional): Count, if any. Defaults to 0.
url_name (str, optional): The description of the url, if any. Defaults to "".
"""
if url == "":
url = "ingen url"
url_name = "ingen url"
if soup != "":
soup = str(soup.prettify())
print(e) # FELSÖKNING
key = datetime.now().strftime("%Y%m%d_%H:%M:%S")
doc = {
"_key": key,
"number": nr,
"error": nr,
"user": str(user),
"error": str(e),
"url": str(url),
"url_name": url_name,
"soup": soup,
"traceback": str(traceback),
}
try:
db.insert_document(
"errors",
doc,
overwrite_mode="update",
silent=True,
)
except Exception as e:
print(e)

787
facebook_reactions.py
Unescape View File

@ -1,787 +0,0 @@
import json
import os
import pickle
import random
import re
import traceback
from datetime import datetime
from getopt import GetoptError, getopt
from getpass import getpass
from sys import argv
from time import sleep
import nacl.secret
import nacl.utils
import requests
import werkzeug
werkzeug.cached_property = werkzeug.utils.cached_property
import robobrowser
from arango import ArangoClient
from bs4 import BeautifulSoup
# import other_pictures # Måste uppdateras
def sleep_(t):
"""
Sover en tid nära den angivna (för att inte sökningarna ska bli för lika varandra)
"""
variation = 4 # Testa olika sovlängder för att inte få användaren blockerad
sleep(t * variation * random.randrange(85, 115, 1) / 100)
if random.randrange(0, 60, 1) == 1:
for s in range(0, 300):
print(f"Sover {300 - s} sekunder till... ", end="\r")
sleep(1)
print()
sleep(random.randrange(0, 10, 1) / 4)
def update_cookie(cookies, profile_name):
""" Uppdaterar cookie för browser """
with open("data/cookie_{}.pkl".format(profile_name), "wb") as f:
pickle.dump(cookies, f)
def write_error(nr, e="", traceback="", soup="", user="", url="", count=0, url_name=""):
"""Skriver info efter error till arango
Args:
nr ([type]): error number
e (str, optional): error. Defaults to "".
traceback (str, optional): The traceback from traceback.format_exc(). Defaults to "".
soup (str, optional): Soup. Defaults to "".
user (str, optional): The user. Defaults to "".
url (str, optional): Url, if any. Defaults to "".
count (int, optional): Count, if any. Defaults to 0.
url_name (str, optional): The description of the url, if any. Defaults to "".
"""
if url == "":
url = "ingen url"
url_name = "ingen url"
if soup != "":
soup = str(soup.prettify())
print(e) # FELSÖKNING
key = datetime.now().strftime("%Y%m%d_%H:%M:%S")
doc = {
"_key": key,
"number": nr,
"error": nr,
"user": str(user),
"error": str(e),
"url": str(url),
"url_name": url_name,
"soup": soup,
"traceback": str(traceback),
}
try:
db.insert_document(
"errors",
doc,
overwrite_mode="update",
silent=True,
)
except Exception as e:
print(e)
def facebook_reactions(user, first=False):
# Fixa url:er osv
if user.username.isnumeric():
user.url = url_bas + "/profile.php?id=" + str(user.username)
user.url_photos = user.url + "&v=photos"
else:
user.username = user.username.replace("/", "")
user.url = url_bas + "/" + user.username
user.url_photos = user.url + "/photos"
if user.username in members_checked:
print('Redan kollat', user.username)
return {"friends": friends_of_user(user.username)}
# Gå till sidan för profilbilder
fb_profile.browser.open(user.url_photos)
sleep_(4)
soup = BeautifulSoup(str(fb_profile.browser.parsed), "lxml")
if (
"""You can't use Facebook because your account, or activity on it, doesn't follow our Community Standards."""
in soup.text
):
print("{} blocked\n".format(fb_profile.name).upper())
return "blocked"
elif 'accept all' in soup.text.lower():
fb_profile.accept_cookies()
fb_profile.browser.open(user.url_photos)
soup = BeautifulSoup(str(fb_profile.browser.parsed), "lxml")
user.name = user.username # Om inte namnet hittas senare
try:
for i in soup.find_all("strong"):
if "Notifications" in str(i):
continue
else:
user.name = i.text.strip()
except Exception as e:
write_error(
6,
e=e,
traceback=traceback.format_exc(),
soup=soup,
user=user.username,
url=user.url_profil_photos,
)
if first == True:
print(soup.prettify())
exit()
print(
"Hämtar reaktioner på profilbilder för {name} ({user})".format(
name=user.name, user=user.username
)
)
# Hitta länk till olika saker hos användarem, inkl facebook-id
user.id = ""
for a in soup.find_all("a", href=True):
if "Profile pictures" in a.text:
user.url_album = url_bas + a["href"] # Länk till album för profilbulder
if "profile_id" in a["href"]:
l = a["href"]
user.id = re.search("\d+", l[l.find("id=") + 3 :]).group(0)
if "Likes" in a.text:
user.url_likes = url_bas + a["href"]
if "About" in a.text:
user.url_about = url_bas + a["href"]
if "Timeline" in a.text:
user.url_timeline = url_bas + a["href"]
if "Cover photos" in a.text:
user.url_coverphotos = url_bas + a["href"]
# Gå till profilbilden (den första som kommer upp när man går till profilen)
if not hasattr(user, "url_album"):
user.url_album = ''
user.add_to_db()
print('Hittar inget album för profilbilder.')
write_error(7, soup=soup, user=user.username, url=user.url_album, url_name='user.url_album')
return None
# ATT GÖRA Här kan andra bilder väljas istället
fb_profile.browser.open(user.url_album)
soup = BeautifulSoup(str(fb_profile.browser.parsed), "lxml")
# Samla alla profilbilder i en lista
url_pics = []
pics = soup.find("div", {"id": "thumbnail_area"})
for i in pics.find_all("a"):
a = i["href"]
url_pics.append(a[: a.find("&id")])
try:
user.profile_pictures = len(url_pics)
except:
user.profile_pictures = 0
# Lägg till profilen till arrango
user.add_to_db()
# Gå igenom alla profilbilder upp till ett maximalt antal
count = 0
if single == True:
max_pic = 1
else:
max_pic = 15
for pic in url_pics:
picture = Picture(user.username)
if count == max_pic:
break
else:
count += 1
picture.url = url_bas + pic
picture.id = str(picture.url[picture.url.find("fbid=") + 5 :])
if picture.id in all_pictures:
print('Redan kollat bild', picture.id)
continue
sleep_(5)
try:
fb_profile.browser.open(picture.url)
except Exception as e: # Fel3
write_error(
3,
e=e,
soup=soup,
user=user.username,
url=picture.url,
url_name="url_pic",
traceback=traceback.format_exc(),
)
update_cookie(fb_profile.browser.session.cookies, fb_profile.name)
# Hitta info om bilden
soup = BeautifulSoup(str(fb_profile.browser.parsed), "lxml")
picture.date = soup.find("abbr").text
# Mer info att lägga in?
# Hämta länkar för bilden att userända sen
for a in soup.find_all("a", href=True):
if all(
[
"reaction" in a["href"],
"reactions" not in a["href"],
"=R" not in a["href"],
]
):
url_reactions = url_bas + str(
a["href"]
) # Länk till reaktionerna för bilden
elif a.text == "Visa i fullständig storlek" or a.text == "View full size":
pic = url_bas + a["href"]
picture.url_full = pic[
: pic.find("&")
] # Den fullständiga adressen till bilden, används som _key i pictures
# Skriv ut vilken bild som behandlas
print(
"Bild {count} av {total}".format(count=count, total=user.profile_pictures),
end="\r",
)
# Hämta reaktioner för bilden
sleep_(3)
fb_profile.browser.open(url_reactions)
update_cookie(fb_profile.browser.session.cookies, fb_profile.name)
soup = BeautifulSoup(str(fb_profile.browser.parsed), "lxml")
try:
for a in soup.find_all("a", {"class": "z ba"}, href=True):
url_limit = a["href"]
picture.no_reactions = re.search(r"total_count=(\d+)", url_limit).group(1)
limit = re.search(r"limit=(\d+)", url_limit).group(1)
except UnboundLocalError:
limit = 999
# Addera bilden till arrango
picture.add_to_db()
url_limit = url_bas + url_limit.replace(
"limit=" + str(limit), "limit=" + str(picture.no_reactions)
)
try:
sleep_(4)
fb_profile.browser.open(url_limit)
update_cookie(fb_profile.browser.session.cookies, fb_profile.name)
soup = BeautifulSoup(str(fb_profile.browser.parsed), "lxml")
# Gå igenom alla som reagerat och för in i arango
for li in soup.find_all("li"):
friend = Friend(user.username)
if single == True:
friend.single = True
if "See more" in li.text:
continue
try:
profile = li.find("h3").find("a")
friend.name = profile.text
friend.url = profile["href"]
if "profile.php" in friend.url:
friend.username = friend.url[friend.url.find("id=") + 3 :]
else:
friend.username = friend.url[friend.url.find("/") + 1 :]
reaction = Reaction(user.username, friend.username, picture.id)
for type in ["Love", "Wow", "Like", "Care", "Sad", "Angry", "Haha"]:
if type in str(li):
reaction.type = type
picture.reactions.append(reaction.get_dict())
# Lägg till vännens profil till arrango
friend.add_to_db()
# Lägg till reaktion till arrango
except AttributeError as e: # Fel1
write_error(
1,
e=e,
soup=soup,
user=user.username,
traceback=traceback.format_exc(),
)
pass
if count == max_pic:
db.collection("picture_reactions").insert_many(
picture.reactions, silent=True, overwrite=True
)
db.collection("picture_reactions").insert_many(picture.reactions, silent=True, overwrite=True)
except Exception as e: # Fel2
write_error(
2,
e=e,
soup=soup,
user=user.username,
url=url_limit,
url_name="url_limit",
traceback=traceback.format_exc(),
)
pass
## ATT GÖRA För att lägga till fler reaktioner om det är få reaktioner på profilbilderna (måste uppdateras)
print()
db.update_document(
{
"_id": "members/" + str(user.username),
"checked": True,
"pictures_checked": user.profile_pictures,
}
)
if first == True:
return {"friends": friends}
else:
pass
def friends_of_user(user):
"""Returnernar userändare som reagerat på user:s bilder"""
cursor = db.aql.execute(
"""
FOR doc IN @@col
FILTER doc._to == @user
RETURN DISTINCT doc._from
""",
bind_vars={"@col": "picture_reactions", "user": "members/" + user},
)
return [doc[8:] for doc in cursor]
def checked_members():
cursor = db.aql.execute(
"""
FOR doc IN @@col
FILTER doc.checked == @bool
RETURN doc._key
""",
bind_vars={"@col": "members", "bool": True},
)
members_checked = set([doc for doc in cursor])
return members_checked
def get_profile(nr):
""" Hämtar profil om det inte gjorts förut """
cursor = db.aql.execute(
"""
FOR doc IN @@col
FILTER doc.in_use == @bool
RETURN doc
""",
bind_vars={"@col": "profiles", "bool": False}
)
profile = cursor.next()
# Skriv till fil att använda sen
with open('data/profile{}.json'.format(nr), 'w') as outfile:
json.dump(profile, outfile)
# Uppdatera dokumentet i arango
profile['in_use'] = True
db.update_document(profile, check_rev=False)
return profile
class Proxies:
def __init__(self):
self.proxies = [
'gb25-wg.socks5.mullvad.net:1080',
'gb26-wg.socks5.mullvad.net:1080',
'gb27-wg.socks5.mullvad.net:1080',
'gb28-wg.socks5.mullvad.net:1080',
'gb29-wg.socks5.mullvad.net:1080'
]
def get_proxie(self):
return self.proxies.pop(random.randrange(0, len(self.proxies), 1))
class Friend:
def __init__(self, user):
self.collection = "members"
self.user = user # The friends friend
self.username = ''
self.url = ''
self.name = ''
self.single = ''
def add_to_db(self):
db.insert_document(
self.collection,
{
"_key": self.username,
"url": url_bas + self.url,
"name": self.name,
},
overwrite_mode="update",
silent=True,
)
class Reaction:
def __init__(self, user, friend_username, picture_id):
self.collection = "picture_reactions"
self.user = user
self.picture_id = picture_id
self.user_name_friend = friend_username
self.type = False
def get_dict(self):
key = str(self.picture_id) + "_" + str(self.user_name_friend)
return {
"_to": "members/" + str(self.user),
"_from": "members/" + str(self.user_name_friend),
"_key": key,
"_id": "picture_reactions/" + key,
"picture": self.picture_id,
"reaction": self.type,
}
class User:
def __init__(self, username):
self.collection = "members"
self.username = str(username)
self.fetched = datetime.now().strftime("%Y%m%d_%H:%M:%S")
self.url_coverphotos = ''
self.id = ''
self.url_likes = ''
self.url_about = ''
self.url_timeline = ''
self.profile_pictures = ''
self.url = ''
self.name = ''
def add_to_db(self):
# Lägg till profilen till arrango
db.insert_document(
self.collection,
{
"_key": self.username,
"url": self.url,
"name": self.name,
"profile_pictures": self.profile_pictures,
"facebook_id": self.id,
"timeline": self.url_timeline,
"likes": self.url_likes,
"about": self.url_about,
"cover photos": self.url_coverphotos,
"fetched": self.fetched
},
overwrite_mode="update",
silent=True,
keep_none=False
)
class Picture:
def __init__(self, user):
self.collection = "pictures"
self.user = user
self.id = ''
self.url_full = ''
self.date = ''
self.url = ''
self.no_reactions = ''
self.reactions = []
def add_to_db(self):
db.insert_document(
self.collection,
{
"_key": self.id,
"url": self.url_full,
"date": self.date,
"url": self.url,
"no_reactions": self.no_reactions,
"user": self.user,
},
overwrite_mode="update",
silent=True,
keep_none=False
)
class Profile:
def __init__(self, nr, new=False):
try:
with open("data/profile{}.json".format(nr)) as f:
self.doc = json.load(f)
except:
self.doc = get_profile(nr)
if 'blocked' in self.doc or new == True:
self.doc = get_profile(nr)
# Användaruppgifter
self.name = self.doc["name"].strip()
self.email = self.doc["email"]
self.pwd = self.doc["pwd"]
self.server = self.doc["server"]
self.nr = nr
# Ange proxies
session = requests.Session()
session.proxies = {
"https": "socks5://'8155249667566524'@{}".format(self.server),
"http": "socks5://'8155249667566524'@{}".format(self.server),
}
# Starta browser
user_agent = "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e Safari/602.1"
self.browser = robobrowser.RoboBrowser(
session=session, user_agent=user_agent, history=False, parser="lxml"
)
try:
self.browser.session.cookies = pickle.load(
open("data/cookie_{}.pkl".format(self.name), "rb")
)
self.logged_in = True
except:
self.logged_in = False
def accept_cookies(self):
""" Accepterar cookies """
self.browser.open("https://mbasic.facebook.com")
soup = BeautifulSoup(str(self.browser.parsed), "lxml")
if 'accept all' not in soup.text.lower():
sleep_(2)
cookie_accept_url = "https://mbasic.facebook.com/cookie/consent-page"
self.browser.open(cookie_accept_url)
sleep_(2)
try:
form = self.browser.get_form()
self.browser.submit_form(form)
print(f"Accepterade cookies för {self.name}")
sleep_(2)
update_cookie(self.browser.session.cookies, self.name)
except Exception as e:
print(f"\nAccepterade inte cookies för {self.name}\n")
def login(self):
""" Loggar in på Facebook. """
print("Loggar in {}\n".format(self.name))
# Gå till log in-sidan
self.browser.open("https://mbasic.facebook.com/login")
# Kolla om browser redan är inloggad
soup = BeautifulSoup(str(self.browser.parsed), "lxml")
if 'log out' in soup.text.lower():
print("Redan inloggad.")
# Hitta och fyll i formulär
form = self.browser.get_form(id="login_form")
form["email"].value = self.email
form["pass"].value = self.pwd
self.browser.submit_form(form, submit=form["login"])
# Vänta lite och uppdatera cookie
print("\nLoggade in\n")
sleep_(2)
def block(self):
""" Blockerar profilen """
if "blocked" not in self.doc:
self.doc["blocked"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
db.update_document(self.doc, silent=True, check_rev=False)
with open("data/profile{}.json".format(self.nr), "w") as outfile:
json.dump(self.doc, outfile)
if __name__ == "__main__":
print()
# Säkerställ att arbetsmappen är samma som den där scriptet ligger
os.chdir(os.path.dirname(__file__))
# Starta koppling till arangodb
# Info för arangodb
user_arango = "Lasse"
pwd_arango = "4c071768bedc259288361c07aafd8535fca546086fada4e7b5de4e2bb26b0e70fa8d348c998b90d032a5b8f3fdbae1881b843021e3475198e6fb45f58d8dc450bd52f77d"
db_arango = "facebook"
host_arango = "http://arango.lasseedfast.se"
# Avkryptera lösen till arango
for i in range(0, 6, 1):
if i == 5:
exit()
try:
key = "sssladnnklja" + getpass()
pwd = (
nacl.secret.SecretBox(key.encode())
.decrypt(pwd_arango, encoder=nacl.encoding.HexEncoder)
.decode("utf-8")
)
break
except:
print("Fel lösenord.")
sleep(1)
client = ArangoClient(hosts=host_arango)
db = client.db(db_arango, username=user_arango, password=pwd)
members = db.collection("members")
pictures = db.collection("pictures")
argv = argv[1:]
try:
opts, args = getopt(argv, "su:", ["single", "user="])
single = True if "-s" in [o[0] for o in opts] else False
for o, a in opts:
if o in ["-u", "--user"]:
users = [
User(str(i).strip())
for i in [(str(i).strip()) for i in a.split(",")]
]
if "users" not in globals():
users = [
User(str(i).strip())
for i in input("Vem/vilka vill du kolla bilder för? ").split(",")
]
except GetoptError:
users = [
User(str(i).strip())
for i in input("Vem/vilka vill du kolla bilder för? ").split(",")
]
single = (
True
if input("Söka bara en bild (single)?").lower() in ["ja, yes, j, y"]
else False
)
print("Kollar profilbilder för:")
for user in users:
print("-", user.username)
print()
# Skapa tre olika profiler att besöka Facebook med
fb_profiles = {}
extra_proxies = Proxies()
for nr in range(1, 4):
fb_profiles[nr] = Profile(nr)
fb_profiles[nr].browser.open('https://api.ipify.org')
soup = BeautifulSoup(str(fb_profiles[nr].browser.parsed), "lxml")
print(soup.text)
if fb_profiles[nr].logged_in == False:
fb_profiles[nr].accept_cookies()
sleep_(2)
fb_profiles[nr].login()
sleep(3)
fb_profile_nr = 1
fb_profile = fb_profiles[fb_profile_nr]
print("Börjar med profilen", fb_profile.name)
url_bas = "https://mbasic.facebook.com"
while True:
for user in users:
# Set för kollade bilder och kollade medlemmar
all_pictures = set([doc["_key"] for doc in pictures.all()])
members_checked = checked_members()
# Hämta reaktioner för den första användaren
facebook_reactions(user, first=True)
friends = friends_of_user(user.username)
friends_unchecked = list(set(friends) - set(members_checked))
# Här följer cookien med så att vi fortfarnade är inloggade
print("\nKlar med", user.username, "\n")
print("Vänner som reagerat:", len(friends))
print("Vänner att kolla:")
for friend in friends_unchecked:
print(friend)
print()
# Hämta reaktioner för den första användarens vänner (som reagerat)
count_friends = 0
for f in friends:
count_friends += 1
user = User(str(f))
sleep_(2)
try:
out = facebook_reactions(user)
if out == "blocked":
# Ta bort profilen ur databasen
db.collection('profiles').delete(fb_profile.doc['_key'], silent=True, ignore_missing=True)
print(
f'{fb_profile.name} blockerad och borttagen {datetime.now().strftime("%Y%m%d_%H:%M:%S")}.'
)
fb_profiles.remove(fb_profile)
try:
# l = [p['nr'] for p in fb_profiles]
# l.sort()
# nr = int(l[-1]+1)
fb_profiles[fb_profile_nr] = Profile(nr, new=True)
print("Laddat ny profil:", fb_profiles[fb_profile_nr].name)
sleep(3)
except e:
print("Det behövs nya profiler...")
for s in range(0, 1600/len(fb_profiles)):
print(f'Sover {600-s} sekunder till... ', end='\r')
fb_profile_nr += 1
print(f"Försöker med {fb_profiles[fb_profile_nr].name}.")
else:
print("Klar med", user.username, "\n")
# Rotera fb-profiler
if count_friends == 6:
if random.randrange(0, 2, 1) == 1:
fb_profile_nr += 1
count_friends = 0
print("Växlar till", fb_profiles[fb_profile_nr].name)
elif count_friends == 10:
fb_profile_nr += 1
count_friends = 0
print("Växlar till", fb_profiles[fb_profile_nr].name)
if fb_profile_nr > len(fb_profiles):
fb_profile_nr = 1
fb_profile = fb_profiles[fb_profile_nr]
except Exception as e: # Fel4
soup = BeautifulSoup(str(fb_profile.browser.parsed), "lxml")
write_error(
4,
e=e,
user=user.username,
traceback=traceback.format_exc(),
soup=soup,
)
print("\nFel: ", str(user.username), "\n")
sleep_(15)
pass
# Ladda in nya användare att kolla
print("\nVem vill du kolla upp?")
users = [User(str(i).strip()) for i in input(">>> ").split(",")]

824
htmlerror.html
Unescape View File

@ -0,0 +1,824 @@
"soup": "
<?xml version=\"1.0\" encoding=\"utf-8\"?>\n
<!DOCTYPE html PUBLIC \"-//WAPFORUM//DTD XHTML Mobile 1.0//EN\" \"http://www.wapforum.org/DTD/xhtml-mobile10.dtd\">\n
<html lang=\"en\" xmlns=\"http://www.w3.org/1999/xhtml\">\n
<head>\n <title>\n Nils Edfast\n </title>\n
<meta content=\"initial-scale=1, width=240\" name=\"viewport\" />\n
<meta content=\"default\" id=\"meta_referrer\" name=\"referrer\" />\n <style nonce=\"aoBhuKKQ\" type=\"text/css\">
\n
/*<![CDATA[*/
.cd {
margin-bottom: 4px;
}
.b .dl {
padding: 0;
}
.b .w {
padding: 2px;
}
.b .bz {
padding: 4px;
}
.bh {
background-color: #fff;
}
.i {
background-color: #3b5998;
}
.e {
background-color: #eceff5;
}
.ct {
background-color: #fffbe2;
color: #7f7212;
}
.j {
padding: 2px 3px;
}
.bi {
padding: 4px 3px;
}
.cu {
border-top: 1px solid;
}
.bj {
border-bottom: 1px solid;
}
.bh {
border-color: #e9e9e9;
}
.i {
border-color: #1d4088;
}
.e {
border-color: #d8dfea;
}
.ct {
border-color: #e2c822;
}
.bk .bl {
display: inline-block;
margin-right: 4px;
}
.b .bk .bl {
float: left;
}
.b .bk .bn {
display: table-cell;
}
.bv {
clear: both;
}
.b a,
.b a:visited {
color: #3b5998;
text-decoration: none;
}
.b .by,
.b .by:visited {
color: #6d84b4;
}
.b .bg,
.b .bg:visited {
color: #fff;
}
.b a:focus,
.b a:hover,
.b .by:focus,
.b .by:hover {
background-color: #3b5998;
color: #fff;
}
.b .bg:focus,
.b .bg:hover {
background-color: #fff;
color: #3b5998;
}
.bm {
background: #eceff5;
}
.bp {
margin: 2px 0 0 5px;
}
.s {
border: 0;
display: inline-block;
vertical-align: top;
}
i.s u {
position: absolute;
width: 0;
height: 0;
overflow: hidden;
}
.ce {
display: block;
font-size: small;
margin-bottom: 4px;
}
.cm {
font-size: small;
font-weight: bold;
margin-top: 4px;
text-align: center;
}
.cc {
border-color: #ccced3 #c4c6ca #b4b6ba;
border-style: solid;
border-width: 1px;
}
.cb {
background: #fff;
}
.ci {
display: inline-block;
margin: 2px;
overflow: hidden;
position: relative;
}
.ci:-moz-focusring {
outline-color: #000;
outline-offset: 1px;
}
.ci:after {
border: 1px solid rgba(0, 0, 0, .1);
bottom: 0;
content: '';
left: 0;
position: absolute;
right: 0;
top: 0;
}
.cf {
text-align: left;
}
.ck {
text-align: center;
}
.cl {
text-align: right;
}
.b .m {
border: 0;
border-collapse: collapse;
margin: 0;
padding: 0;
width: 100%;
}
.b .m tbody {
vertical-align: top;
}
.b .cq>tr>td,
.b .cq>tbody>tr>td,
.b .m td.cq {
vertical-align: middle;
}
.b .m td {
padding: 0;
}
.b .m td.w {
padding: 2px;
}
.b .m td.bz {
padding: 4px;
}
.b .t {
width: 100%;
}
.cj {
background: #f2f2f2;
}
.cg:hover .cj {
background: none;
}
.bs .bt {
display: block;
}
.bt {
border: solid 2px;
cursor: pointer;
margin: 0;
padding: 2px 6px 3px;
text-align: center;
}
.bu,
.b a.bu {
background: #f3f4f5;
border-color: #ccc #aaa #999;
color: #505c77;
}
.i .bu,
.b .i a.bu {
background: #3b5998;
border-color: #8a9ac5 #29447e #1a356e;
color: #fff;
}
.bt .s {
pointer-events: none;
}
.bt {
display: inline-block;
}
.bt+.bt {
margin-left: 3px;
}
.bt input {
background: none;
border: none;
margin: 0;
padding: 0;
}
.bu input {
color: #505c77;
}
.i .bu input {
color: #fff;
}
.co {
border-bottom: 1px solid #e5e5e5;
display: block;
font-size: small;
padding: 4px;
}
.cs {
border-top: 1px solid #e5e5e5;
font-size: small;
font-weight: bold;
padding: 4px;
text-align: center;
}
.cp>*,
.cp.cp>* {
border-bottom: 1px solid #e5e5e5;
}
.cp>:last-child {
border-bottom: none;
}
.cp+.cp {
border-top: 1px solid #e5e5e5;
}
.bx {
color: gray;
}
.cr {
font-size: small;
}
body,
tr,
input,
textarea,
.f {
font-size: medium;
}
body {
text-align: left;
direction: ltr;
}
body,
tr,
input,
textarea,
button {
font-family: sans-serif;
}
body,
p,
figure,
h1,
h2,
h3,
h4,
h5,
h6,
ul,
ol,
li,
dl,
dd,
dt {
margin: 0;
padding: 0;
}
h1,
h2,
h3,
h4,
h5,
h6 {
font-size: 1em;
font-weight: bold;
}
ul,
ol {
list-style: none;
}
article,
aside,
figcaption,
figure,
footer,
header,
nav,
section {
display: block;
}
#page {
position: relative;
}
.r,
.r.s {
display: block;
}
.o {
display: block;
}
.p {
height: 20px;
width: 20px;
}
.k {
background: #3b5998;
padding: 0 4px 4px;
height: 22px;
}
.k.k .x {
background: #fff;
border: 1px solid #243872;
box-sizing: border-box;
font-size: small;
height: 22px;
margin: 0;
width: 100%;
}
.l.k {
padding: 1px 1px 3px;
}
.k .q {
padding: 1px 3px 0 0;
}
.k.k.k .bc {
background: #627aba;
border: 1px solid #2e417e;
color: #fff;
font-size: x-small;
font-weight: normal;
height: 22px;
line-height: 20px;
margin-left: 3px;
}
form {
margin: 0;
border: 0;
}
.v {
border: 0;
display: block;
margin: 0;
padding: 0;
}
.bb {
-webkit-appearance: none;
background: none;
display: inline-block;
font-size: 12px;
height: 28px;
line-height: 28px;
margin: 0;
overflow: visible;
padding: 0 9px;
text-align: center;
vertical-align: top;
white-space: nowrap;
}
.b .bb {
border-radius: 2px;
}
.bd,
a.bd,
html .b a.bd {
color: #fff;
}
.b .bd {
background-color: #4267b2;
border: 1px solid #365899;
}
.b a.bd:hover,
.b .bd:hover {
background-color: #465e91;
}
.bd[disabled] {
color: #899bc1;
}
.b .bd[disabled]:hover {
background-color: #4267b2;
}
.b a.bb::after {
content: \"\";
display: inline-block;
height: 100%;
vertical-align: middle;
}
.b .bb {
padding: 0 8px;
}
.b a.bb {
height: 26px;
line-height: 26px;
}
.ba {
font-weight: normal;
}
.dc {
font-size: small;
padding: 7px 8px 8px;
}
.dq {
border: 1px solid;
border-color: #8d949e;
border-radius: 4px;
display: block;
margin-top: 8px;
padding: 4px;
text-align: center;
}
.cz {
display: block;
font-size: x-small;
margin: -3px -3px 1px -3px;
padding: 3px;
}
.db {
border: 1px solid #90949c;
display: block;
font-size: x-large;
height: 20px;
line-height: 18px;
text-align: center;
vertical-align: middle;
width: 20px;
}
.b .dc td.cx {
padding-right: 4px;
}
.b .dc td.da {
padding-left: 4px;
}
.dc.dd {
background-color: #444950;
}
.dd {
border-top: 1px solid #444950;
color: #bec3c9;
}
.b .dd a,
.b .dd a:visited,
.b .cw input,
.b .cw input:visited,
.b .cw a,
.b .cw a:visited {
color: #bec3c9;
}
.b .dd a:focus,
.b .dd a:hover,
.b .cw input:focus,
.b .cw input:hover,
.b .cw a:focus,
.b .cw a:hover {
background: #dadde1;
color: #1d2129;
}
.df {
margin-bottom: 8px;
}
.dc.dd .df>table {
background: #d3d7dc;
border: 1px solid #444950;
}
.dm {
background: #d3d7dc;
}
.cv {
background-color: #444950;
font-size: x-small;
padding: 7px 8px 8px;
}
.cy {
color: #bec3c9;
display: block;
font-size: x-small;
margin: -3px -3px 1px -3px;
padding: 3px;
}
.de .dp {
height: 24px;
line-height: 24px;
margin-left: 2px;
}
.dh {
background: #fff;
}
.de .dn {
background-color: transparent;
color: #4b4f56;
display: block;
padding: 0;
width: 100%;
}
.di .s {
display: block;
}
.de .dg .do {
padding: 2px;
}
.de .dg .di {
padding: 4px;
}
.b .de .dg {
border: 1px solid #8d949e;
}
.be {
padding-bottom: 1px;
}
.bf {
display: inline-block;
font-size: small;
padding: 2px 4px 2px;
}
/*]]>*/
\n
</style>\n
<link crossorigin=\"use-credentials\" href=\"/data/manifest/\" rel=\"manifest\" />\n
</head>\n
<body class=\"b c d e\" tabindex=\"0\">\n <div class=\"f\">\n <div id=\"viewport\">\n <div class=\"g h\"
id=\"MChromeHeader\">\n <div class=\"i j\" id=\"header\" role=\"banner\">\n <form action=\"/search/\"
class=\"k l\" method=\"get\">\n <input name=\"search\" type=\"hidden\" value=\"Search\" />\n
<input name=\"search_source\" type=\"hidden\" value=\"top_nav\" />\n <table class=\"m\"
role=\"presentation\">\n <tbody>\n <tr>\n <td class=\"n\">\n <a class=\"o p q\"
href=\"/home.php?ref_component=mbasic_home_logo&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
<img alt=\"Facebook logo\" class=\"r s\" height=\"20\"
src=\"https://static.xx.fbcdn.net/rsrc.php/v3/yu/r/CEFRKDb4iKw.png\"
width=\"20\" />\n </a>\n </td>\n <td class=\"t u\">\n <input
autocomplete=\"off\" autocorrect=\"off\" class=\"v w x\" name=\"query\"
placeholder=\"Search Facebook\" spellcheck=\"false\" type=\"text\" />\n
</td>\n <td class=\"n y\">\n <input class=\"z ba bb bc bd\" type=\"submit\"
value=\"Search\" />\n </td>\n </tr>\n </tbody>\n </table>\n </form>\n <div
class=\"be\" role=\"navigation\">\n <a class=\"bf bg\"
href=\"/home.php?ref_component=mbasic_home_header&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Home\n </a>\n <a class=\"bf bg\"
href=\"/tina.shiawi?v=info&amp;ref_component=mbasic_home_header&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Edit Profile\n </a>\n <a accesskey=\"3\" class=\"bf bg\"
href=\"/notifications.php?ref_component=mbasic_home_header&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Notifications\n </a>\n <a accesskey=\"2\" class=\"bf bg\"
href=\"/friends/center/mbasic/?fb_ref=tn&amp;sr=1&amp;ref_component=mbasic_home_header&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Find Friends\n </a>\n <a class=\"bf bg\"
href=\"/pages/?ref_component=mbasic_home_header&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Pages\n </a>\n <a class=\"bf bg\"
href=\"/groups/?category=membership&amp;ref_component=mbasic_home_header&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Groups\n </a>\n <a class=\"bf bg\"
href=\"/coronavirus_info/?page_source=tab&amp;ref_component=mbasic_home_header&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
COVID-19\n </a>\n <a accesskey=\"5\" class=\"bf bg\"
href=\"/menu/bookmarks/?ref_component=mbasic_home_header&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Menu\n </a>\n </div>\n </div>\n </div>\n <div id=\"objects_container\">\n <div class=\"e\"
id=\"root\" role=\"main\">\n <div class=\"bh bi bj\">\n <div class=\"bk\">\n <a class=\"bl\"
href=\"/profile/picture/view/?profile_id=861795563\" id=\"u_0_0_b7\">\n <img alt=\"Nils
Edfast, profile picture\" class=\"bm s\" height=\"72\"
src=\"https://scontent-lhr8-1.xx.fbcdn.net/v/t1.0-1/cp0/e15/q65/p320x320/11393254_10155641291180564_4453469323237941935_n.jpg?_nc_cat=100&amp;ccb=1-3&amp;_nc_sid=dbb9e7&amp;efg=eyJpIjoiYiJ9&amp;_nc_ohc=HxPSvefBW9UAX_V4c-Q&amp;_nc_ht=scontent-lhr8-1.xx&amp;tp=3&amp;oh=43bd216f1bc9e7b1bfb6e947acd344f5&amp;oe=60790106\"
width=\"72\" />\n </a>\n <div class=\"bn\">\n <span>\n <strong class=\"bo\">\n Nils
Edfast\n </strong>\n <img aria-label=\"Nils Edfast is available on his phone\"
class=\"bp bq s\" height=\"12\"
src=\"https://static.xx.fbcdn.net/rsrc.php/v3/y7/r/FbBF_Z_7R_O.png\"
width=\"8\" />\n </span>\n <br />\n <div class=\"br\">\n <table class=\"bs\">\n
<tr>\n <td>\n <a class=\"bt bu\"
href=\"/a/mobile/friends/profile_add_friend.php?subjectid=861795563&amp;istimeline=1&amp;hf=profile_button&amp;fref=unknown&amp;frefid=0&amp;gfid=AQBeyEkTq-_kRp7G-KM\">\n
Add Friend\n </a>\n </td>\n <td>\n <a class=\"bt bu\"
href=\"/messages/thread/861795563/?entrypoint=profile_message_button\">\n
Message\n </a>\n </td>\n <td>\n <a class=\"bt bu\"
href=\"/mbasic/more/?owner_id=861795563\">\n More\n </a>\n </td>\n
</tr>\n </table>\n </div>\n </div>\n <div class=\"bv\">\n </div>\n </div>\n <div
class=\"bw f bx\">\n <a class=\"by\"
href=\"/nils.edfast?v=timeline&amp;lst=100064870566411%3A861795563%3A1616049760\">\n
Timeline\n </a>\n ·\n <a class=\"by\"
href=\"/nils.edfast/about?lst=100064870566411%3A861795563%3A1616049760\">\n About\n
</a>\n ·\n <a class=\"by\"
href=\"/nils.edfast?v=likes&amp;lst=100064870566411%3A861795563%3A1616049760\">\n
Likes\n </a>\n </div>\n </div>\n <div class=\"bz ca\">\n <div class=\"cb bz cc cd\"
title=\"Uploads\">\n <h3 class=\"ce\">\n Uploads\n </h3>\n <div>\n <table class=\"m\"
role=\"presentation\">\n <tbody>\n <tr>\n <td class=\"t cf\" style=\"width:33%\">\n
<a class=\"cg ch ci\"
href=\"/photo.php?fbid=10156598623075564&amp;id=861795563&amp;set=pb.861795563.-2207520000..&amp;source=9\">\n
<img class=\"cj s\" height=\"70\"
src=\"https://scontent-lhr8-1.xx.fbcdn.net/v/t31.0-0/cp0/e15/q65/c97.0.160.160a/p160x160/12823462_10156598623075564_1251308470166199557_o.jpg?_nc_cat=110&amp;ccb=1-3&amp;_nc_sid=05277f&amp;efg=eyJpIjoiYiJ9&amp;_nc_ohc=DDD-n1lHa5QAX8EIOg6&amp;_nc_ht=scontent-lhr8-1.xx&amp;tp=5&amp;oh=6fd23f07c9b489a71ae4e1f318e88c9a&amp;oe=6076B82C\"
width=\"70\" />\n </a>\n </td>\n <td class=\"t ck\"
style=\"width:33%\">\n <a class=\"cg ch ci\"
href=\"/photo.php?fbid=10155183058095564&amp;id=861795563&amp;set=pb.861795563.-2207520000..&amp;source=9\">\n
<img class=\"cj s\" height=\"70\"
src=\"https://scontent-lht6-1.xx.fbcdn.net/v/t31.0-0/cp0/e15/q65/c31.0.160.160a/p160x160/1966267_10155183058095564_2339870776965752430_o.jpg?_nc_cat=111&amp;ccb=1-3&amp;_nc_sid=05277f&amp;efg=eyJpIjoiYiJ9&amp;_nc_ohc=Fs6bfBzKydgAX9SI2Zb&amp;tn=-8t6HwLdXVJSx-Oy&amp;_nc_ht=scontent-lht6-1.xx&amp;tp=5&amp;oh=8f35d7ce9a0e2aa97f120f99f8b58117&amp;oe=6078F178\"
width=\"70\" />\n </a>\n </td>\n <td class=\"t cl\"
style=\"width:33%\">\n <a class=\"cg ch ci\"
href=\"/photo.php?fbid=10154773588360564&amp;id=861795563&amp;set=pb.861795563.-2207520000..&amp;source=9\">\n
<img class=\"cj s\" height=\"70\"
src=\"https://scontent-lht6-1.xx.fbcdn.net/v/t31.0-0/cp0/e15/q65/c53.0.160.160a/p160x160/10688374_10154773588360564_5983645517190919804_o.jpg?_nc_cat=106&amp;ccb=1-3&amp;_nc_sid=dd9801&amp;efg=eyJpIjoiYiJ9&amp;_nc_ohc=FaA-LDk-lQsAX8hBv77&amp;_nc_ht=scontent-lht6-1.xx&amp;tp=5&amp;oh=8be2cd4ab81ea1337d8368ae3e52d362&amp;oe=60793FB8\"
width=\"70\" />\n </a>\n </td>\n </tr>\n </tbody>\n </table>\n
</div>\n <div class=\"cm\">\n <a
href=\"/nils.edfast/photoset/pb.861795563.-2207520000../?owner_id=861795563&amp;back_uri=%2Fnils.edfast%2Fphotos%2F\">\n
See All\n </a>\n </div>\n </div>\n <div class=\"cb cc cn cd\">\n <h3 class=\"co\">\n
Albums\n </h3>\n <div>\n <ul class=\"cp\">\n <li class=\"bz\">\n <table class=\"m cq\"
role=\"presentation\">\n <tbody>\n <tr>\n <td class=\"t\">\n <span
class=\"cr\">\n <a
href=\"/nils.edfast/albums/103961505563/\">\n
In-flytta-ut-elsedagsfest\n </a>\n </span>\n </td>\n
</tr>\n </tbody>\n </table>\n </li>\n <li class=\"bz\">\n <table
class=\"m cq\" role=\"presentation\">\n <tbody>\n <tr>\n <td class=\"t\">\n
<span class=\"cr\">\n <a
href=\"/nils.edfast/albums/115137500563/\">\n Jul, ett
naket album\n </a>\n </span>\n </td>\n </tr>\n </tbody>
\n </table>\n </li>\n <li class=\"bz\">\n <table class=\"m cq\"
role=\"presentation\">\n <tbody>\n <tr>\n <td class=\"t\">\n <span
class=\"cr\">\n <a
href=\"/nils.edfast/albums/5800565563/\">\n Hultsfred
2k6\n </a>\n </span>\n </td>\n </tr>\n </tbody>\n
</table>\n </li>\n <li class=\"bz\">\n <table class=\"m cq\"
role=\"presentation\">\n <tbody>\n <tr>\n <td class=\"t\">\n <span
class=\"cr\">\n <a
href=\"/nils.edfast/albums/5800570563/\">\n De gammla
bilderna från lucksta i ungdomens år\n </a>\n </span>\n
</td>\n </tr>\n </tbody>\n </table>\n </li>\n <li class=\"bz\">\n
<table class=\"m cq\" role=\"presentation\">\n <tbody>\n <tr>\n <td class=\"t\">
\n <span class=\"cr\">\n <a
href=\"/nils.edfast/albums/10152062796760564/\">\n Cover
photos\n </a>\n </span>\n </td>\n </tr>\n </tbody>\n
</table>\n </li>\n </ul>\n </div>\n <div class=\"cs\">\n <a
href=\"/nils.edfast/photos/albums/?owner_id=861795563\">\n See All (10)\n </a>\n
</div>\n </div>\n </div>\n <div style=\"display:none\">\n <div>\n <iframe frameborder=\"0\"
height=\"1\" marginheight=\"0\" marginwidth=\"0\" scrolling=\"no\"
src=\"/sem_pixel/1/test/0/\" width=\"1\">\n </iframe>\n <iframe frameborder=\"0\"
height=\"1\" marginheight=\"0\" marginwidth=\"0\" scrolling=\"no\"
src=\"/sem_pixel/1/test/1/\" width=\"1\">\n </iframe>\n </div>\n <div>\n </div>\n <div>
\n </div>\n <div>\n </div>\n </div>\n </div>\n </div>\n <div class=\"cb\">\n <div class=\"ct
bi cu\">\n <a href=\"/download.php?cr=int_mf\">\n Install Facebook on your iPhone and browse
faster\n </a>\n </div>\n <div class=\"cv\">\n <div class=\"cw\">\n <table class=\"m\"
role=\"presentation\">\n <tbody>\n <tr>\n <td class=\"t cx\" style=\"width:50%\">\n <b
class=\"cy\">\n English (UK)\n </b>\n <a class=\"cz\"
href=\"/a/language.php?l=pl_PL&amp;lref=https%3A%2F%2Fmbasic.facebook.com%2Fnils.edfast%2Fphotos&amp;index=2&amp;sref=legacy_mbasic_footer&amp;gfid=AQBTI3rwp6E57zLRqj0&amp;ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Polski\n </a>\n <a class=\"cz\"
href=\"/a/language.php?l=pt_BR&amp;lref=https%3A%2F%2Fmbasic.facebook.com%2Fnils.edfast%2Fphotos&amp;index=4&amp;sref=legacy_mbasic_footer&amp;gfid=AQA8Zl6L8nnENAHqXG4&amp;ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Português (Brasil)\n </a>\n </td>\n <td class=\"t da\" style=\"width:50%\">
\n <a class=\"cz\"
href=\"/a/language.php?l=en_US&amp;lref=https%3A%2F%2Fmbasic.facebook.com%2Fnils.edfast%2Fphotos&amp;index=1&amp;sref=legacy_mbasic_footer&amp;gfid=AQCuT1z0ZkUG9rH1BJY&amp;ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
English (US)\n </a>\n <a class=\"cz\"
href=\"/a/language.php?l=es_LA&amp;lref=https%3A%2F%2Fmbasic.facebook.com%2Fnils.edfast%2Fphotos&amp;index=3&amp;sref=legacy_mbasic_footer&amp;gfid=AQDAYh6ihljlGHBBqSM&amp;ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Español\n </a>\n <a class=\"cz\"
href=\"/language.php?n=https%3A%2F%2Fmbasic.facebook.com%2Fnils.edfast%2Fphotos&amp;ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
<div class=\"db\">\n +\n </div>\n </a>\n </td>\n </tr>\n </tbody>\n </table>
\n </div>\n </div>\n <div class=\"dc dd\">\n <div id=\"search_div\">\n <form action=\"/search/\"
method=\"get\">\n <input name=\"search\" type=\"hidden\" />\n <input name=\"search_source\"
type=\"hidden\" value=\"footer\" />\n <div class=\"de df\">\n <table class=\"m dg dh\"
role=\"presentation\">\n <tbody>\n <tr>\n <td class=\"n di cq\">\n <label
for=\"u_0_1_pP\">\n <img class=\"dj s\" height=\"20\"
role=\"presentation\"
src=\"https://static.xx.fbcdn.net/rsrc.php/v3/yB/r/RlIWdLhwVjw.png\"
width=\"20\" />\n </label>\n </td>\n <td class=\"t dk cq\">\n
<input aria-label=\"Search\" class=\"v dl dm dn\" id=\"u_0_1_pP\"
name=\"query\" type=\"text\" />\n </td>\n <td class=\"n do cq\">\n
<input class=\"z ba bb dp bd\" type=\"submit\" value=\"Search\" />\n
</td>\n </tr>\n </tbody>\n </table>\n </div>\n </form>\n </div>\n <table
class=\"m\" role=\"presentation\">\n <tbody>\n <tr>\n <td class=\"t cx\" style=\"width:50%\">\n
<a class=\"cz\"
href=\"/pages/create/?ref_type=site_footer&amp;ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Create Page\n </a>\n <a accesskey=\"0\" class=\"cz\"
href=\"/help/?ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Help\n </a>\n <a accesskey=\"7\" class=\"cz\"
href=\"/settings/?entry_point=mbasic_footer_link&amp;ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Settings &amp; privacy\n </a>\n </td>\n <td class=\"t da\" style=\"width:50%\">
\n <a class=\"cz\"
href=\"/bugnub/?source=Footer&amp;ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Report a Problem\n </a>\n <a accesskey=\"8\" class=\"cz\"
href=\"/policies/?ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\">\n
Terms &amp; Policies\n </a>\n <a class=\"cz\"
href=\"/login/save-password-interstitial/?ref_component=mbasic_footer&amp;ref_page=%2Fwap%2Fprofile_timeline.php%3Aphotos\"
id=\"mbasic_logout_button\">\n Log Out (Tina Shiawi)\n </a>\n </td>\n </tr>\n
</tbody>\n </table>\n <a class=\"dq\" href=\"#header\">\n Back to Top\n </a>\n </div>\n </div>\n
</div>\n </div>\n </body>\n
</html>",

267
scrapers.py
Unescape View File

@ -0,0 +1,267 @@
from classes import Picture, Friend, Reaction
from helpers import sleep_, write_error, update_cookie
from config import *
import traceback
import re
from arangodb import db
def profile_picture_reactions(profile, user, all_pictures, first=False, single = False):
# Fixa url:er osv
if user.username.isnumeric():
user.url = url_bas + "/profile.php?id=" + str(user.username)
user.url_photos = user.url + "&v=photos"
else:
user.username = user.username.replace("/", "")
user.url = url_bas + "/" + user.username
user.url_photos = user.url + "/photos"
# Gå till sidan för profilbilder
profile.browser.open(user.url_photos)
sleep_(4)
if (
"""You can't use Facebook because your account, or activity on it, doesn't follow our Community Standards."""
in profile.viewing().text
):
print("{} blocked\n".format(profile.name).upper())
profile.blocked = True
return None
elif 'accept all' in profile.viewing().text.lower():
profile.accept_cookies()
profile.browser.open(user.url_photos)
user.name = user.username # Om inte namnet hittas senare
try:
for i in profile.viewing().find_all("strong"):
if "Notifications" in str(i):
continue
else:
user.name = i.text.strip()
except Exception as e:
write_error(
6,
e=e,
traceback=traceback.format_exc(),
soup=profile.viewing(),
user=user.username,
url=user.url_photos,
)
if first == True:
print(profile.viewing().prettify())
exit()
print(
"Hämtar reaktioner på profilbilder för {name} ({user})".format(
name=user.name, user=user.username
)
)
# Hitta länk till olika saker hos användarem, inkl facebook-id
user.id = ""
for a in profile.viewing().find_all("a", href=True):
if "Profile pictures" in a.text:
user.url_album = url_bas + a["href"] # Länk till album för profilbulder
if "profile_id" in a["href"]:
l = a["href"]
user.id = re.search("\d+", l[l.find("id=") + 3 :]).group(0)
if "Likes" in a.text:
user.url_likes = url_bas + a["href"]
if "About" in a.text:
user.url_about = url_bas + a["href"]
if "Timeline" in a.text:
user.url_timeline = url_bas + a["href"]
if "Cover photos" in a.text:
user.url_coverphotos = url_bas + a["href"]
user.add_to_db()
# Gå till profilbilden (den första som kommer upp när man går till profilen)
if not hasattr(user, "url_album"):
write_error(9, soup=profile.viewing(), user=user.username)
if user.url_other_picture != '':
# Använd eventuell extrabild och ta bort den från användaren
url_pics = [user.url_other_picture]
user.url_other_picture = ''
else:
# Spara ner profilen till databasen och avsluta sökningen på användaren
user.url_album = False
if first == False:
user.doc['checked'] = True
user.add_to_db()
print('Hittar inget album för profilbilder.')
write_error(7, soup=profile.viewing(), user=user.username, url=user.url_album, url_name='user.url_album')
return None
# ATT GÖRA Här kan andra bilder väljas istället
else:
profile.browser.open(user.url_album)
# Samla alla profilbilder i en lista
url_pics = []
pics = profile.viewing().find("div", {"id": "thumbnail_area"})
for i in pics.find_all("a"):
a = i["href"]
url_pics.append(a[: a.find("&id")])
if user.url_other_picture != '':
# Lägg till eventuell extrabild och ta bort den från användaren
url_pics.append(user.url_other_picture)
user.url_other_picture = ''
try:
user.profile_pictures = len(url_pics)
except:
user.profile_pictures = 0
user.doc['checked'] = True
user.add_to_db()
return
# Lägg till profilen till arrango
user.add_to_db()
# Gå igenom alla profilbilder
if single == True and first == False:
url_pics = url_pics[0]
for pic in url_pics:
# Skriv ut vilken bild som behandlas
print(f"Bild {url_pics.index(pic) + 1} av {user.profile_pictures}", end="\r",)
picture = Picture(user.username)
picture.url = url_bas + pic
picture.id = str(picture.url[picture.url.find("fbid=") + 5 :])
picture.id = str(re.search('\d+', picture.id).group())
# if picture.id in all_pictures:
# print('Redan kollat bild', picture.id)
# continue
sleep_(5)
try:
profile.browser.open(picture.url)
except Exception as e: # Fel3
write_error(
3,
e=e,
soup=profile.viewing(),
user=user.username,
url=picture.url,
url_name="url_pic",
traceback=traceback.format_exc(),
)
update_cookie(profile.browser.session.cookies, profile.name)
# Hitta info om bilden
try:
picture.date = profile.viewing().find("abbr").text
except Exception as e: # Fel8
write_error(8, e=e, soup=profile.viewing(), url=pic, url_name='picture url', user=user.name, traceback=traceback.format_exc())
# ATT GÖRA Mer info att lägga in?
# Hämta länkar för bilden att userända sen
#print(profile.viewing().prettify())
for a in profile.viewing().find_all("a", href=True):
if all(
[
"reaction" in a["href"],
"reactions" not in a["href"],
"=R" not in a["href"],
]
):
url_reactions = url_bas + str(a["href"]) # Länk till reaktionerna för bilden
elif a.text == "View full size":
pic = url_bas + a["href"]
picture.url_full = pic[
: pic.find("&")
] # Den fullständiga adressen till bilden, används som _key i pictures
if 'url_reactions' not in globals():
for a in profile.viewing().find_all("a", href=True):
if '/likes/' in a["href"]:
url_reactions = url_bas + str(a["href"])
if 'url_reactions' not in globals():
for div in profile.viewing().find_all("div", href=True):
if 'like this' in div.text:
url_reactions = url_bas + str(div["href"])
# Hämta reaktioner för bilden
sleep_(3)
profile.browser.open(url_reactions)
update_cookie(profile.browser.session.cookies, profile.name)
try:
for a in profile.viewing().find_all("a", {"class": "z ba"}, href=True):
url_limit = a["href"]
picture.no_reactions = re.search(r"total_count=(\d+)", url_limit).group(1)
limit = re.search(r"limit=(\d+)", url_limit).group(1)
except UnboundLocalError:
limit = 999
# Addera bilden till arrango
picture.add_to_db()
url_limit = url_bas + url_limit.replace(
"limit=" + str(limit), "limit=" + str(picture.no_reactions)
)
try:
sleep_(4)
profile.browser.open(url_limit)
update_cookie(profile.browser.session.cookies, profile.name)
# Gå igenom alla som reagerat och för in i arango
for li in profile.viewing().find_all("li"):
friend = Friend(user.username)
if single == True:
friend.single = True
if "See more" in li.text:
continue
try:
friend_html = li.find("h3").find("a")
friend.name = friend_html.text
friend.url = friend_html["href"]
if "profile.php" in friend.url:
friend.username = friend.url[friend.url.find("id=") + 3 :]
else:
friend.username = friend.url[friend.url.find("/") + 1 :]
reaction = Reaction(user.username, friend.username, picture.id)
for type in ["Love", "Wow", "Like", "Care", "Sad", "Angry", "Haha"]:
if type in str(li):
reaction.type = type
picture.reactions.append(reaction.get_dict())
# Lägg till vännens profil till arrango
friend.add_to_db()
# Lägg till reaktion till arrango
except AttributeError as e: # Fel1
write_error(
1,
e=e,
soup=profile.viewing(),
user=user.username,
traceback=traceback.format_exc(),
)
pass
if count == max_pic:
db.collection("picture_reactions").insert_many(
picture.reactions, silent=True, overwrite=True
)
db.collection("picture_reactions").insert_many(picture.reactions, silent=True, overwrite=True)
except Exception as e: # Fel2
write_error(
2,
e=e,
soup=profile.viewing(),
user=user.username,
url=url_limit,
url_name="url_limit",
traceback=traceback.format_exc(),
)
pass
## ATT GÖRA För att lägga till fler reaktioner om det är få reaktioner på profilbilderna (måste uppdateras)
user.checked()
Write Preview
Loading…
Cancel
Save