113 lines
3.3 KiB
Python
113 lines
3.3 KiB
Python
import io
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import requests
|
|
|
|
KAIKKI_TURKISH_URL = "https://kaikki.org/dictionary/Turkish/kaikki.org-dictionary-Turkish.jsonl"
|
|
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
DATA_DIR = os.path.join(BASE_DIR, "data")
|
|
DB_PATH = os.path.join(DATA_DIR, "lex.db")
|
|
|
|
|
|
def download_and_ingest_turkish():
|
|
print(f"Downloading Turkish lexicon from {KAIKKI_TURKISH_URL}...")
|
|
response = requests.get(KAIKKI_TURKISH_URL, stream=True)
|
|
response.raise_for_status()
|
|
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cur = conn.cursor()
|
|
|
|
# Create tr_entries table
|
|
cur.execute("DROP TABLE IF EXISTS tr_entries;")
|
|
cur.execute("""
|
|
CREATE TABLE tr_entries (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
word TEXT NOT NULL,
|
|
pos TEXT NOT NULL,
|
|
glosses TEXT NOT NULL,
|
|
examples TEXT NOT NULL,
|
|
synonyms TEXT NOT NULL
|
|
);
|
|
""")
|
|
|
|
cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_word ON tr_entries (word);")
|
|
cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_pos ON tr_entries (pos);")
|
|
|
|
insert_sql = """
|
|
INSERT INTO tr_entries (word, pos, glosses, examples, synonyms)
|
|
VALUES (?, ?, ?, ?, ?);
|
|
"""
|
|
|
|
batch = []
|
|
total_count = 0
|
|
|
|
print("Parsing and inserting Turkish entries into SQLite...")
|
|
for line in response.iter_lines():
|
|
if not line:
|
|
continue
|
|
try:
|
|
item = json.loads(line.decode("utf-8"))
|
|
except Exception:
|
|
continue
|
|
|
|
word = item.get("word", "").strip().lower()
|
|
pos = item.get("pos", "").strip().lower()
|
|
if not word or not pos:
|
|
continue
|
|
|
|
senses = item.get("senses", [])
|
|
glosses = []
|
|
examples = []
|
|
synonyms = []
|
|
|
|
for s in senses:
|
|
for g in s.get("glosses", []):
|
|
g_clean = g.strip()
|
|
if g_clean and g_clean not in glosses:
|
|
glosses.append(g_clean)
|
|
|
|
for ex in s.get("examples", []):
|
|
text = ex.get("text", "").strip()
|
|
eng_trans = ex.get("english", "").strip()
|
|
if text:
|
|
if eng_trans:
|
|
examples.append(f"{text} -> {eng_trans}")
|
|
else:
|
|
examples.append(text)
|
|
|
|
for syn in s.get("synonyms", []):
|
|
syn_word = syn.get("word", "").strip()
|
|
if syn_word and syn_word not in synonyms:
|
|
synonyms.append(syn_word)
|
|
|
|
if not glosses:
|
|
continue
|
|
|
|
batch.append((
|
|
word,
|
|
pos,
|
|
json.dumps(glosses, ensure_ascii=False),
|
|
json.dumps(examples, ensure_ascii=False),
|
|
json.dumps(synonyms, ensure_ascii=False),
|
|
))
|
|
total_count += 1
|
|
|
|
if len(batch) >= 5000:
|
|
cur.executemany(insert_sql, batch)
|
|
conn.commit()
|
|
batch = []
|
|
print(f"Inserted {total_count} Turkish entries...")
|
|
|
|
if batch:
|
|
cur.executemany(insert_sql, batch)
|
|
conn.commit()
|
|
|
|
conn.close()
|
|
print(f"Turkish lexicon ingestion complete. Total entries: {total_count}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
download_and_ingest_turkish()
|