lasse
/
mala
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

.

Browse Source
main
Lasse Server 2 years ago
parent 4d17f8f3af
commit c6becadd82
2 changed files with 24 additions and 44 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 49
      extract_persons.py
  2. 19
      test.py

49
extract_persons.py
Unescape View File

@ -18,8 +18,6 @@ def check_name(name, persons):
return valid_name return valid_name
def execute_query_with_retry(db, query, max_retries=5, delay=2): def execute_query_with_retry(db, query, max_retries=5, delay=2):
for attempt in range(max_retries): for attempt in range(max_retries):
try: try:
@ -36,6 +34,7 @@ def execute_query_with_retry(db, query, max_retries=5, delay=2):
# Then, in your extract_persons function: # Then, in your extract_persons function:
def extract_persons(interrogation): def extract_persons(interrogation):
sleep(random.uniform(0.05, 0.3))
print(interrogation["_key"].upper()) print(interrogation["_key"].upper())
q = "for doc in persons filter doc.other != true return doc" q = "for doc in persons filter doc.other != true return doc"
result = execute_query_with_retry(db, q) result = execute_query_with_retry(db, q)
@ -131,17 +130,17 @@ def extract_persons(interrogation):
if interrogation["_key"] not in person["mentioned_in_interrogation"]: if interrogation["_key"] not in person["mentioned_in_interrogation"]:
person["mentioned_in_interrogation"].append(interrogation["_key"]) person["mentioned_in_interrogation"].append(interrogation["_key"])
db.collection("persons").update(person, check_rev=False) db.collection("persons").update(person, check_rev=False)
db.collection("all_relations").insert( # db.collection("all_relations").insert(
{ # {
"_from": interrogation["person_id"], # "_from": interrogation["person_id"],
"_to": person["_id"], # "_to": person["_id"],
"relation": "mentioned_by", # "relation": "mentioned_by",
"mentions": [{'interrogation': interrogation["_key"], "date": interrogation["date"], "mentioned_as": info}], # "mentions": [{'interrogation': interrogation["_key"], "date": interrogation["date"], "mentioned_as": info}],
"_key": f'{interrogation["_key"]}_{person["_key"]}' # "_key": f'{interrogation["_key"]}_{person["_key"]}'
}, # },
overwrite_mode="update", # overwrite_mode="update",
merge=True, # merge=True,
) # )
else: else:
print(f"\033[91m{name} not found in database\033[0m") print(f"\033[91m{name} not found in database\033[0m")
@ -158,18 +157,18 @@ def extract_persons(interrogation):
merge=True, merge=True,
) )
db.collection("all_relations").insert( # db.collection("all_relations").insert(
{ # {
"_from": interrogation["person_id"], # "_from": interrogation["person_id"],
"_to": doc["_id"], # "_to": doc["_id"],
"relation": "mentioned_by", # "relation": "mentioned_by",
'other': True, # 'other': True,
"mentions": [{'interrogation': interrogation["_key"], "date": interrogation["date"], "mentioned_as": info}], # "mentions": [{'interrogation': interrogation["_key"], "date": interrogation["date"], "mentioned_as": info}],
"_key": f'{interrogation["_key"]}_{doc["_key"]}' # "_key": f'{interrogation["_key"]}_{doc["_key"]}'
}, # },
overwrite_mode="update", # overwrite_mode="update",
merge=True, # merge=True,
) # )
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
print(f"\033[91mError when processing {name}: {e}\033[0m") print(f"\033[91mError when processing {name}: {e}\033[0m")

19
test.py
Unescape View File

@ -1,19 +0,0 @@
from _arango import arango
# arango.db.collection("other_persons").truncate()
# arango.db.collection("all_relations").truncate()
# persons = list(arango.db.collection("persons").all())
# for person in persons:
# for interrogation in person['interrogations']:
# arango.db.collection('interrogations').update({'_key': interrogation, 'person_id': person['_id']}, )
# print(f"Updated {interrogation} with person_id {person['_id']}")
interrogations = list(arango.db.collection("interrogations").all())
for interrogation in interrogations:
interrogation['person_id'] = 'persons/' + interrogation['person'].replace('persons_', '')
arango.db.collection('interrogations').update(interrogation, merge=False)
Write Preview
Loading…
Cancel
Save