lasse
/
fb-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/lasseedfast/fb-scraper

Browse Source
master
Lasse Edfast 4 years ago
parent df39ff447b 6d24628bb9
commit f69de458f1
5 changed files with 86 additions and 52 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 17
      .gitignore
  2. 15
      docker/stats/Dockerfile
  3. 27
      docker/stats/requirements.txt
  4. 44
      docker/stats/stats.py
  5. 31
      requirements.txt

17
.gitignore vendored
Unescape View File

@ -1,18 +1,27 @@
# Blandat
/.DS_Store /.DS_Store
/.venv *.venv
/.vscode /.vscode
/__pycache__ /__pycache__
*.json *.json
*.pkl *.pkl
facebook/test.py
/data/* /data/*
*.html *.html
*.code-workspace *.code-workspace
workspace.code-workspace workspace.code-workspace
password_arango.txt password_arango.txt
*.gexf *.gexf
facebook/mrkoll.
*.pyc *.pyc
*.sqlite3
#facebook
/facebook /facebook
!/facebook/*.py !/facebook/*.py
*.sqlite3 facebook/test.py
facebook/mrkoll.
# docker
/stats/*
!/stats/*.py

15
docker/stats/Dockerfile
Unescape View File

@ -0,0 +1,15 @@
FROM python:alpine
WORKDIR /
RUN apk add --update --no-cache g++ gcc libxslt-dev
COPY requirements.txt .
RUN pip install -r requirements.txt
ADD . .
ENTRYPOINT [ "python", "stats.py" ]
# docker buildx build --file docker/stats/Dockerfile --platform linux/arm64,linux/amd64 -t mrkoll .

27
docker/stats/requirements.txt
Unescape View File

@ -0,0 +1,27 @@
black==21.8b0
certifi==2020.6.20
chardet==4.0.0
click==8.0.1
httplib2==0.18.1
idna==2.10
mypy-extensions==0.4.3
packaging==21.0
pathspec==0.9.0
platformdirs==2.3.0
#pycurl==7.43.0.6
PyJWT==2.1.0
pyparsing==2.4.7
PySimpleSOAP==1.16.2
#python-apt==2.2.1
python-arango==7.2.0
python-debian==0.1.39
python-debianbts==3.1.0
regex==2021.8.28
#reportbug==7.10.3
requests==2.25.1
requests-toolbelt==0.9.1
setuptools-scm==6.3.1
six==1.16.0
tomli==1.2.1
typing-extensions==3.10.0.2
urllib3==1.26.5

44
facebook/stats.py → docker/stats/stats.py
Unescape View File

@ -3,21 +3,20 @@ from getpass import getpass
from time import sleep from time import sleep
from arango import ArangoClient from arango import ArangoClient
from json2html import json2html
def now(): def now():
""" Returns current date and time as string""" """Returns current date and time as string"""
return datetime.now().strftime("%Y-%m-%d_%H:%M:%S") return datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
def write_stats(db, continuous=False): def write_stats(db, continuous=False):
while True: while True:
d = {} d = {}
for col in db.collections(): for col in db.collections():
if not col['system']: if not col["system"]:
d[col['name']] = db.collection(col['name']).count() d[col["name"]] = db.collection(col["name"]).count()
del d['stats'] del d["stats"]
#d['time'] = now() # d['time'] = now()
cursor = db.aql.execute( cursor = db.aql.execute(
""" """
FOR doc IN members FOR doc IN members
@ -26,49 +25,42 @@ def write_stats(db, continuous=False):
RETURN length RETURN length
""" """
) )
d['checked_members'] = cursor.next() d["checked_members"] = cursor.next()
# Hur många konton per säljare som finns kvar # Hur många konton per säljare som finns kvar
cursor = db.aql.execute( cursor = db.aql.execute(
''' """
for doc in profiles for doc in profiles_webshare
filter has(doc, "vendor") filter has(doc, "vendor")
COLLECT vendor = doc.vendor WITH COUNT INTO length COLLECT vendor = doc.vendor WITH COUNT INTO length
RETURN { RETURN {
"vendor" : vendor, "vendor" : vendor,
"active" : length "active" : length
} }
''') """
d['active_vendors'] = [doc for doc in cursor] )
d["active_vendors"] = [doc for doc in cursor]
d['_key'] = now()[:13]
db.insert_document( "stats", d, overwrite=True)
# Skriv en html-fil
with open('website/fb-webbapp/stats.html', 'a+') as html:
html.truncate(0)
html.write('<!DOCTYPE html> <br>')
html.write(json2html.convert(json = d)) d["_key"] = now()[:13]
db.insert_document("stats", d, overwrite=True)
# Sov för att fortsätta senare # Sov för att fortsätta senare
if continuous: if continuous:
print(now())
sleep(86400) sleep(86400)
else: else:
break break
# Info för arangodb # Info för arangodb
user_arango = "Stats" user_arango = "Stats"
db_arango = "facebook" db_arango = "facebook"
host_arango = "http://192.168.0.4:8529" host_arango = "http://192.168.1.10:8529"
# Starta koppling till arangodb # Starta koppling till arangodb
# Avkryptera lösen till arango # Avkryptera lösen till arango
pwd = getpass(f'Arangolösenord för {user_arango}:').strip() pwd = getpass(f"Arangolösenord för {user_arango}:").strip()
db = ArangoClient(hosts=host_arango).db( db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)
db_arango, username=user_arango, password=pwd
)
write_stats(db, continuous=True) write_stats(db, continuous=True)

31
requirements.txt
Unescape View File

@ -1,21 +1,12 @@
beautifulsoup4==4.9.3 certifi==2021.5.30
certifi==2020.12.5 charset-normalizer==2.0.4
cffi==1.14.5 idna==3.2
chardet==4.0.0 packaging==21.0
idna==2.10 PyJWT==2.1.0
lxml pyparsing==2.4.7
pycparser==2.20 python-arango==7.2.0
PyJWT==2.0.1 requests==2.26.0
#PyNaCl==1.4.0
PySocks==1.7.1
python-arango==7.1.0
requests==2.25.1
requests-toolbelt==0.9.1 requests-toolbelt==0.9.1
robobrowser==0.5.3 setuptools-scm==6.3.1
setuptools-scm==5.0.2 tomli==1.2.1
six==1.15.0 urllib3==1.26.6
soupsieve==2.2
toml==0.10.2
urllib3==1.26.3
Werkzeug==1.0.1
json2html

Write Preview
Loading…
Cancel
Save