lasse
/
fb-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/lasseedfast/fb-scraper

Browse Source
master
Lasse Server 4 years ago
parent c3941f634c 17ea7c96c9
commit 46f95ddadf
5 changed files with 540 additions and 3 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 81
      fb-webbapp/app.py
  2. 19
      fb-webbapp/config.py
  3. 14
      fb-webbapp/html_page.py
  4. 7
      mrkoll/mrkoll_oxylabs.py
  5. 422
      mrkoll/servers_oxylabs.py

81
fb-webbapp/app.py
Unescape View File

@ -0,0 +1,81 @@
from flask import Flask, render_template, redirect, url_for, request
from time import sleep
from sshtunnel import open_tunnel
import paramiko
from getpass import getpass
from arango import ArangoClient
from config import host_adress, db_arango
from html_page import html_text
# Route for handling the login page logic
# import the Flask class from the flask module
from flask import Flask, render_template
# create the application object
app = Flask(__name__)
# use decorators to link the function to a url
@app.route("/")
def home():
return "Hello, World!" # return a string
@app.route("/welcome")
def welcome():
return render_template("welcome.html") # render a template
@app.route("/result")
def result(url):
return render_template(url) # render a template
@app.route("/login", methods=["GET", "POST"])
def login():
error = None
if request.method == "POST":
credentials = {
"lasse": "test"
}
if (
request.form["username"] not in credentials
or request.form["password"] != credentials[request.form["username"]]
):
sleep(2)
error = "Fel lösenord. Testa igen."
else:
sleep(2)
username = 'Webbapp'
pwd = 'jagged-CUTTER-lyon-yourself-scavenge'
path_to_key="/Users/Lasse/.ssh/id_rsa"
pwd_key = getpass('Key pwd: ')
with open_tunnel(
(host_adress, 2210),
ssh_username="lasse",
ssh_pkey=paramiko.RSAKey.from_private_key_file(
path_to_key, password=pwd_key
),
ssh_private_key_password=pwd_key,
remote_bind_address=("127.0.0.1", 8529)) as server:
port_arango = server.local_bind_port # En annan port än den "vanliga", ska användas nedan.
db = ArangoClient(hosts=f'{"http://127.0.0.1"}:{port_arango}').db(
db_arango, username=username, password=pwd,
)
r = db.collection('members').random()
html = html_text.replace('RESULTAT', r['_key'])
with open('/Users/Lasse/Datorgemensamt/Programmeringsprojekt/Facebook/fb-scraper/fb-webbapp/templates/test.html', 'w') as page:
page.write(html)
# Gör söking, skapa sida, returnera den.a
return render_template('test.html')
return render_template("login.html", error=error)
# start the server with the 'run()' method
if __name__ == "__main__":
app.run(debug=False)

19
fb-webbapp/config.py
Unescape View File

@ -0,0 +1,19 @@
from getpass import getpass
def set_pwd(_pwd=None):
global pwd
if _pwd == None:
_pwd = getpass('Lösenord för Arango-användaren:')
pwd = _pwd
# Info för arangodb
user_arango = "Lasse"
db_arango = "facebook"
host_arango = 'http://192.168.1.10'
port_arango = '8529'
host_adress = "studio-garda.asuscomm.com" #IP/adress till där db finns
# Andra uppgifter
url_bas = "https://mbasic.facebook.com"
user_agent = "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e Safari/602.1"
mullvad = '8155249667566524'

14
fb-webbapp/html_page.py
Unescape View File

@ -0,0 +1,14 @@
html_text = """<!DOCTYPE html>
<html>
<head>
<title>Resultat</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<body>
<div class="container">
<br>
<p>RESULTAT</p>
</div>
</body>
</html>"""

7
mrkoll/mrkoll_oxylabs.py
Unescape View File

@ -14,6 +14,7 @@ import urllib
import urllib.request as request import urllib.request as request
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from arango import ArangoClient from arango import ArangoClient
from servers_oxylabs import servers
def find_person(number, errors, server): def find_person(number, errors, server):
@ -26,7 +27,7 @@ def find_person(number, errors, server):
url = f'https://mrkoll.se/resultat?n={number}' url = f'https://mrkoll.se/resultat?n={number}'
user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36" user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36"
headers = {'User-agent': user_agent} headers = {'User-agent': user_agent}
query = request.build_opener(request.ProxyHandler({'http': proxy})) query = request.build_opener(request.ProxyHandler({'https': proxy}))
req = request.Request(url, headers=headers) req = request.Request(url, headers=headers)
n = 0 n = 0
@ -163,13 +164,13 @@ if __name__ == "__main__":
# proxies = { 'https': 'https://il061376:"typical-humidify-upheave-aback-rusty"@lexvpn.integrity.st:1723' } # proxies = { 'https': 'https://il061376:"typical-humidify-upheave-aback-rusty"@lexvpn.integrity.st:1723' }
servers_json = json.load(open('oxylabs_servers_trail.json')) servers_json = servers
# Info för arangodb # Info för arangodb
user_arango = "Phone" user_arango = "Phone"
db_arango = "facebook" db_arango = "facebook"
host_arango = "http://192.168.1.20:8529" host_arango = "http://192.168.1.10:8529"
# Starta koppling till arangodb # Starta koppling till arangodb
# Avkryptera lösen till arango # Avkryptera lösen till arango

422
mrkoll/servers_oxylabs.py
Unescape View File

@ -0,0 +1,422 @@
servers = [
{
"ip": "149.71.186.1",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.101",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.112",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.114",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.116",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.118",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.124",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.129",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.131",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.133",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.144",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.146",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.148",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.150",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.156",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.16",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.161",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.163",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.165",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.176",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.178",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.18",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.180",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.188",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.193",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.195",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.197",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.20",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.208",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.210",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.212",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.22",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.220",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.225",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.227",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.229",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.240",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.242",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.244",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.252",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.3",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.33",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.35",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.37",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.48",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.5",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.50",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.52",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.54",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.60",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.65",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.67",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.69",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.80",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.82",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.84",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.86",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.92",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.97",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
{
"ip": "149.71.186.99",
"port": "60000",
"country": "SE",
"city": "Stockholm",
"created_at": "2021-09-09 12:48:31",
},
]
Write Preview
Loading…
Cancel
Save