From 9f21a1abaea0126537362d79d59d39e3997b5c70 Mon Sep 17 00:00:00 2001 From: Lasse Server Date: Wed, 8 Sep 2021 14:47:23 +0200 Subject: [PATCH 1/5] . --- .gitignore | 2 ++ docker/free/Dockerfile | 19 ----------- facebook/stats.py | 74 ------------------------------------------ requirements.txt | 31 +++++++----------- 4 files changed, 13 insertions(+), 113 deletions(-) delete mode 100644 docker/free/Dockerfile delete mode 100644 facebook/stats.py diff --git a/.gitignore b/.gitignore index a9f76ee..6fb1199 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,7 @@ password_arango.txt facebook/mrkoll. *.pyc /facebook +/docker +!/docker/*.py !/facebook/*.py *.sqlite3 \ No newline at end of file diff --git a/docker/free/Dockerfile b/docker/free/Dockerfile deleted file mode 100644 index c289f9d..0000000 --- a/docker/free/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ - -FROM python:3.8 - -WORKDIR / - -COPY requirements.txt . - -RUN pip install -r requirements.txt - -ADD . . - -ENTRYPOINT [ "python", "facebook/__main__.py", "-p free" ] - -CMD ["",""] - -# BUILD: -# docker buildx create --use -#docker buildx build --file docker/free/Dockerfile --platform linux/arm -t l3224/fb-scraper:free --push . - diff --git a/facebook/stats.py b/facebook/stats.py deleted file mode 100644 index 60764f2..0000000 --- a/facebook/stats.py +++ /dev/null @@ -1,74 +0,0 @@ -from datetime import datetime -from getpass import getpass -from time import sleep - -from arango import ArangoClient -from json2html import json2html - - -def now(): - """ Returns current date and time as string""" - return datetime.now().strftime("%Y-%m-%d_%H:%M:%S") - -def write_stats(db, continuous=False): - while True: - d = {} - for col in db.collections(): - if not col['system']: - d[col['name']] = db.collection(col['name']).count() - del d['stats'] - #d['time'] = now() - cursor = db.aql.execute( - """ - FOR doc IN members - FILTER doc.checked == true - COLLECT WITH COUNT INTO length - RETURN length - """ - ) - d['checked_members'] = cursor.next() - - - # Hur många konton per säljare som finns kvar - cursor = db.aql.execute( - ''' - for doc in profiles - filter has(doc, "vendor") - COLLECT vendor = doc.vendor WITH COUNT INTO length - RETURN { - "vendor" : vendor, - "active" : length - } - ''') - d['active_vendors'] = [doc for doc in cursor] - - d['_key'] = now()[:13] - db.insert_document( "stats", d, overwrite=True) - - # Skriv en html-fil - with open('website/fb-webbapp/stats.html', 'a+') as html: - html.truncate(0) - html.write('
') - - html.write(json2html.convert(json = d)) - - # Sov för att fortsätta senare - if continuous: - sleep(86400) - else: - break - -# Info för arangodb -user_arango = "Stats" -db_arango = "facebook" -host_arango = "http://192.168.0.4:8529" - -# Starta koppling till arangodb -# Avkryptera lösen till arango -pwd = getpass(f'Arangolösenord för {user_arango}:').strip() - -db = ArangoClient(hosts=host_arango).db( - db_arango, username=user_arango, password=pwd -) - -write_stats(db, continuous=True) diff --git a/requirements.txt b/requirements.txt index 574069e..c610c7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,12 @@ -beautifulsoup4==4.9.3 -certifi==2020.12.5 -cffi==1.14.5 -chardet==4.0.0 -idna==2.10 -lxml -pycparser==2.20 -PyJWT==2.0.1 -#PyNaCl==1.4.0 -PySocks==1.7.1 -python-arango==7.1.0 -requests==2.25.1 +certifi==2021.5.30 +charset-normalizer==2.0.4 +idna==3.2 +packaging==21.0 +PyJWT==2.1.0 +pyparsing==2.4.7 +python-arango==7.2.0 +requests==2.26.0 requests-toolbelt==0.9.1 -robobrowser==0.5.3 -setuptools-scm==5.0.2 -six==1.15.0 -soupsieve==2.2 -toml==0.10.2 -urllib3==1.26.3 -Werkzeug==1.0.1 -json2html +setuptools-scm==6.3.1 +tomli==1.2.1 +urllib3==1.26.6 From 8bb207ebfcd25a783b8ca1b264910161fb1c012c Mon Sep 17 00:00:00 2001 From: Lasse Server Date: Wed, 8 Sep 2021 14:48:30 +0200 Subject: [PATCH 2/5] . --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6fb1199..8324173 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ password_arango.txt facebook/mrkoll. *.pyc /facebook -/docker +/docker/* !/docker/*.py !/facebook/*.py *.sqlite3 \ No newline at end of file From 64eb32245b05138e39a397c21dff8ac7062d7819 Mon Sep 17 00:00:00 2001 From: Lasse Server Date: Wed, 8 Sep 2021 14:53:43 +0200 Subject: [PATCH 3/5] Made it cleaner --- .gitignore | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 8324173..e25c77b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,27 @@ + +# Blandat /.DS_Store -/.venv +*.venv /.vscode /__pycache__ *.json *.pkl -facebook/test.py /data/* *.html *.code-workspace workspace.code-workspace password_arango.txt *.gexf -facebook/mrkoll. *.pyc +*.sqlite3 + +#facebook /facebook +!/facebook/*.py +facebook/test.py +facebook/mrkoll. + +# docker /docker/* !/docker/*.py -!/facebook/*.py -*.sqlite3 \ No newline at end of file + From 91046da67fd333c09668a3ebe6dd6f9421b703a7 Mon Sep 17 00:00:00 2001 From: Lasse Server Date: Wed, 8 Sep 2021 14:55:49 +0200 Subject: [PATCH 4/5] . --- .gitignore | 4 +-- docker/stats/Dockerfile | 13 +++++++ docker/stats/requirements.txt | 27 ++++++++++++++ docker/stats/stats.py | 66 +++++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 docker/stats/Dockerfile create mode 100644 docker/stats/requirements.txt create mode 100644 docker/stats/stats.py diff --git a/.gitignore b/.gitignore index e25c77b..5169577 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,6 @@ facebook/test.py facebook/mrkoll. # docker -/docker/* -!/docker/*.py +/stats/* +!/stats/*.py diff --git a/docker/stats/Dockerfile b/docker/stats/Dockerfile new file mode 100644 index 0000000..8cd6c9a --- /dev/null +++ b/docker/stats/Dockerfile @@ -0,0 +1,13 @@ +FROM python:alpine + +WORKDIR / + +COPY requirements.txt . + +RUN pip install -r requirements.txt + +ADD docker/stats/. . + +ENTRYPOINT [ "python", "facebook/mrkoll.py" ] + +# docker buildx build --file docker/stats/Dockerfile --platform linux/arm64,linux/amd64 -t mrkoll . \ No newline at end of file diff --git a/docker/stats/requirements.txt b/docker/stats/requirements.txt new file mode 100644 index 0000000..2ba3112 --- /dev/null +++ b/docker/stats/requirements.txt @@ -0,0 +1,27 @@ +black==21.8b0 +certifi==2020.6.20 +chardet==4.0.0 +click==8.0.1 +httplib2==0.18.1 +idna==2.10 +mypy-extensions==0.4.3 +packaging==21.0 +pathspec==0.9.0 +platformdirs==2.3.0 +pycurl==7.43.0.6 +PyJWT==2.1.0 +pyparsing==2.4.7 +PySimpleSOAP==1.16.2 +python-apt==2.2.1 +python-arango==7.2.0 +python-debian==0.1.39 +python-debianbts==3.1.0 +regex==2021.8.28 +reportbug==7.10.3 +requests==2.25.1 +requests-toolbelt==0.9.1 +setuptools-scm==6.3.1 +six==1.16.0 +tomli==1.2.1 +typing-extensions==3.10.0.2 +urllib3==1.26.5 diff --git a/docker/stats/stats.py b/docker/stats/stats.py new file mode 100644 index 0000000..d4b8f92 --- /dev/null +++ b/docker/stats/stats.py @@ -0,0 +1,66 @@ +from datetime import datetime +from getpass import getpass +from time import sleep + +from arango import ArangoClient + + +def now(): + """Returns current date and time as string""" + return datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + +def write_stats(db, continuous=False): + while True: + d = {} + for col in db.collections(): + if not col["system"]: + d[col["name"]] = db.collection(col["name"]).count() + del d["stats"] + # d['time'] = now() + cursor = db.aql.execute( + """ + FOR doc IN members + FILTER doc.checked == true + COLLECT WITH COUNT INTO length + RETURN length + """ + ) + d["checked_members"] = cursor.next() + + # Hur många konton per säljare som finns kvar + cursor = db.aql.execute( + """ + for doc in profiles_webshare + filter has(doc, "vendor") + COLLECT vendor = doc.vendor WITH COUNT INTO length + RETURN { + "vendor" : vendor, + "active" : length + } + """ + ) + d["active_vendors"] = [doc for doc in cursor] + + d["_key"] = now()[:13] + db.insert_document("stats", d, overwrite=True) + + # Sov för att fortsätta senare + if continuous: + print(now()) + sleep(86400) + else: + break + + +# Info för arangodb +user_arango = "Stats" +db_arango = "facebook" +host_arango = "http://192.168.1.10:8529" + +# Starta koppling till arangodb +# Avkryptera lösen till arango +pwd = getpass(f"Arangolösenord för {user_arango}:").strip() + +db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd) + +write_stats(db, continuous=True) From 6d24628bb967c6e0acba818f7214f857d99b903d Mon Sep 17 00:00:00 2001 From: Lasse Server Date: Wed, 8 Sep 2021 15:15:36 +0200 Subject: [PATCH 5/5] Removed requirements and added apk add --- docker/stats/Dockerfile | 6 ++++-- docker/stats/requirements.txt | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/stats/Dockerfile b/docker/stats/Dockerfile index 8cd6c9a..f90f8ac 100644 --- a/docker/stats/Dockerfile +++ b/docker/stats/Dockerfile @@ -2,12 +2,14 @@ FROM python:alpine WORKDIR / +RUN apk add --update --no-cache g++ gcc libxslt-dev + COPY requirements.txt . RUN pip install -r requirements.txt -ADD docker/stats/. . +ADD . . -ENTRYPOINT [ "python", "facebook/mrkoll.py" ] +ENTRYPOINT [ "python", "stats.py" ] # docker buildx build --file docker/stats/Dockerfile --platform linux/arm64,linux/amd64 -t mrkoll . \ No newline at end of file diff --git a/docker/stats/requirements.txt b/docker/stats/requirements.txt index 2ba3112..e392166 100644 --- a/docker/stats/requirements.txt +++ b/docker/stats/requirements.txt @@ -8,16 +8,16 @@ mypy-extensions==0.4.3 packaging==21.0 pathspec==0.9.0 platformdirs==2.3.0 -pycurl==7.43.0.6 +#pycurl==7.43.0.6 PyJWT==2.1.0 pyparsing==2.4.7 PySimpleSOAP==1.16.2 -python-apt==2.2.1 +#python-apt==2.2.1 python-arango==7.2.0 python-debian==0.1.39 python-debianbts==3.1.0 regex==2021.8.28 -reportbug==7.10.3 +#reportbug==7.10.3 requests==2.25.1 requests-toolbelt==0.9.1 setuptools-scm==6.3.1