Merge branch 'master' of https://github.com/lasseedfast/fb-scraper

4 years ago · f69de458f1
parent df39ff447b 6d24628bb9
commit f69de458f1
5 changed files with 86 additions and 52 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,18 +1,27 @@
+
+# Blandat
 /.DS_Store
-/.venv
+*.venv
 /.vscode
 /__pycache__
 *.json
 *.pkl
-facebook/test.py
 /data/*
 *.html
 *.code-workspace
 workspace.code-workspace
 password_arango.txt
 *.gexf
-facebook/mrkoll.
 *.pyc
+*.sqlite3
+
+#facebook
 /facebook
 !/facebook/*.py
-*.sqlite3
+facebook/test.py
+facebook/mrkoll.
+
+# docker
+/stats/*
+!/stats/*.py
+
--- a/docker/stats/Dockerfile
+++ b/docker/stats/Dockerfile
@ -0,0 +1,15 @@
+FROM python:alpine
+
+WORKDIR /
+
+RUN apk add --update --no-cache g++ gcc libxslt-dev
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt
+
+ADD . .
+
+ENTRYPOINT [ "python", "stats.py" ]
+
+# docker buildx build --file docker/stats/Dockerfile --platform linux/arm64,linux/amd64 -t mrkoll .
--- a/docker/stats/requirements.txt
+++ b/docker/stats/requirements.txt
@ -0,0 +1,27 @@
+black==21.8b0
+certifi==2020.6.20
+chardet==4.0.0
+click==8.0.1
+httplib2==0.18.1
+idna==2.10
+mypy-extensions==0.4.3
+packaging==21.0
+pathspec==0.9.0
+platformdirs==2.3.0
+#pycurl==7.43.0.6
+PyJWT==2.1.0
+pyparsing==2.4.7
+PySimpleSOAP==1.16.2
+#python-apt==2.2.1
+python-arango==7.2.0
+python-debian==0.1.39
+python-debianbts==3.1.0
+regex==2021.8.28
+#reportbug==7.10.3
+requests==2.25.1
+requests-toolbelt==0.9.1
+setuptools-scm==6.3.1
+six==1.16.0
+tomli==1.2.1
+typing-extensions==3.10.0.2
+urllib3==1.26.5
--- a/docker/stats/stats.py
+++ b/docker/stats/stats.py
@ -3,21 +3,20 @@ from getpass import getpass
 from time import sleep

 from arango import ArangoClient
-from json2html import json2html


 def now():
-    """ Returns current date and time as string"""
+    """Returns current date and time as string"""
    return datetime.now().strftime("%Y-%m-%d_%H:%M:%S")

 def write_stats(db, continuous=False):
    while True:
        d = {}
        for col in db.collections():
-            if not col['system']:
-                d[col['name']] = db.collection(col['name']).count()
-        del d['stats']
-        #d['time'] = now()
+            if not col["system"]:
+                d[col["name"]] = db.collection(col["name"]).count()
+        del d["stats"]
+        # d['time'] = now()
        cursor = db.aql.execute(
            """
            FOR doc IN members
@ -25,50 +24,43 @@ def write_stats(db, continuous=False):
            COLLECT WITH COUNT INTO length
            RETURN length
                    """
-                )
-        d['checked_members'] = cursor.next()
-
+        )
+        d["checked_members"] = cursor.next()

        # Hur många konton per säljare som finns kvar
        cursor = db.aql.execute(
-        '''
-        for doc in profiles
+            """
+        for doc in profiles_webshare
        filter has(doc, "vendor")
        COLLECT vendor = doc.vendor WITH COUNT INTO length
        RETURN { 
        "vendor" : vendor, 
        "active" : length 
        }
-        ''')
-        d['active_vendors'] = [doc for doc in cursor]
-
-        d['_key'] = now()[:13]
-        db.insert_document( "stats", d, overwrite=True)
+        """
+        )
+        d["active_vendors"] = [doc for doc in cursor]

-        # Skriv en html-fil
-        with open('website/fb-webbapp/stats.html', 'a+') as html:
-            html.truncate(0)
-            html.write('<!DOCTYPE html> <br>')
-
-            html.write(json2html.convert(json = d))
+        d["_key"] = now()[:13]
+        db.insert_document("stats", d, overwrite=True)

        # Sov för att fortsätta senare
        if continuous:
+            print(now())
            sleep(86400)
        else:
            break

+
 # Info för arangodb
 user_arango = "Stats"
 db_arango = "facebook"
-host_arango = "http://192.168.0.4:8529"
+host_arango = "http://192.168.1.10:8529"

 # Starta koppling till arangodb
 # Avkryptera lösen till arango
-pwd = getpass(f'Arangolösenord för {user_arango}:').strip()
+pwd = getpass(f"Arangolösenord för {user_arango}:").strip()

-db = ArangoClient(hosts=host_arango).db(
-    db_arango, username=user_arango, password=pwd
-)
+db = ArangoClient(hosts=host_arango).db(db_arango, username=user_arango, password=pwd)

 write_stats(db, continuous=True)
--- a/requirements.txt
+++ b/requirements.txt
@ -1,21 +1,12 @@
-beautifulsoup4==4.9.3
-certifi==2020.12.5
-cffi==1.14.5
-chardet==4.0.0
-idna==2.10
-lxml
-pycparser==2.20
-PyJWT==2.0.1
-#PyNaCl==1.4.0
-PySocks==1.7.1
-python-arango==7.1.0
-requests==2.25.1
+certifi==2021.5.30
+charset-normalizer==2.0.4
+idna==3.2
+packaging==21.0
+PyJWT==2.1.0
+pyparsing==2.4.7
+python-arango==7.2.0
+requests==2.26.0
 requests-toolbelt==0.9.1
-robobrowser==0.5.3
-setuptools-scm==5.0.2
-six==1.15.0
-soupsieve==2.2
-toml==0.10.2
-urllib3==1.26.3
-Werkzeug==1.0.1
-json2html
+setuptools-scm==6.3.1
+tomli==1.2.1
+urllib3==1.26.6