diff --git a/extract_persons.py b/extract_persons.py index 310e93a..369e1dc 100644 --- a/extract_persons.py +++ b/extract_persons.py @@ -18,8 +18,6 @@ def check_name(name, persons): return valid_name - - def execute_query_with_retry(db, query, max_retries=5, delay=2): for attempt in range(max_retries): try: @@ -36,6 +34,7 @@ def execute_query_with_retry(db, query, max_retries=5, delay=2): # Then, in your extract_persons function: def extract_persons(interrogation): + sleep(random.uniform(0.05, 0.3)) print(interrogation["_key"].upper()) q = "for doc in persons filter doc.other != true return doc" result = execute_query_with_retry(db, q) @@ -131,17 +130,17 @@ def extract_persons(interrogation): if interrogation["_key"] not in person["mentioned_in_interrogation"]: person["mentioned_in_interrogation"].append(interrogation["_key"]) db.collection("persons").update(person, check_rev=False) - db.collection("all_relations").insert( - { - "_from": interrogation["person_id"], - "_to": person["_id"], - "relation": "mentioned_by", - "mentions": [{'interrogation': interrogation["_key"], "date": interrogation["date"], "mentioned_as": info}], - "_key": f'{interrogation["_key"]}_{person["_key"]}' - }, - overwrite_mode="update", - merge=True, - ) + # db.collection("all_relations").insert( + # { + # "_from": interrogation["person_id"], + # "_to": person["_id"], + # "relation": "mentioned_by", + # "mentions": [{'interrogation': interrogation["_key"], "date": interrogation["date"], "mentioned_as": info}], + # "_key": f'{interrogation["_key"]}_{person["_key"]}' + # }, + # overwrite_mode="update", + # merge=True, + # ) else: print(f"\033[91m{name} not found in database\033[0m") @@ -158,18 +157,18 @@ def extract_persons(interrogation): merge=True, ) - db.collection("all_relations").insert( - { - "_from": interrogation["person_id"], - "_to": doc["_id"], - "relation": "mentioned_by", - 'other': True, - "mentions": [{'interrogation': interrogation["_key"], "date": interrogation["date"], "mentioned_as": info}], - "_key": f'{interrogation["_key"]}_{doc["_key"]}' - }, - overwrite_mode="update", - merge=True, - ) + # db.collection("all_relations").insert( + # { + # "_from": interrogation["person_id"], + # "_to": doc["_id"], + # "relation": "mentioned_by", + # 'other': True, + # "mentions": [{'interrogation': interrogation["_key"], "date": interrogation["date"], "mentioned_as": info}], + # "_key": f'{interrogation["_key"]}_{doc["_key"]}' + # }, + # overwrite_mode="update", + # merge=True, + # ) except Exception as e: traceback.print_exc() print(f"\033[91mError when processing {name}: {e}\033[0m") diff --git a/test.py b/test.py deleted file mode 100644 index dc2fbb7..0000000 --- a/test.py +++ /dev/null @@ -1,19 +0,0 @@ -from _arango import arango - -# arango.db.collection("other_persons").truncate() -# arango.db.collection("all_relations").truncate() - -# persons = list(arango.db.collection("persons").all()) - -# for person in persons: -# for interrogation in person['interrogations']: - -# arango.db.collection('interrogations').update({'_key': interrogation, 'person_id': person['_id']}, ) -# print(f"Updated {interrogation} with person_id {person['_id']}") - - -interrogations = list(arango.db.collection("interrogations").all()) - -for interrogation in interrogations: - interrogation['person_id'] = 'persons/' + interrogation['person'].replace('persons_', '') - arango.db.collection('interrogations').update(interrogation, merge=False) \ No newline at end of file