Files
lex/scripts/build_turkish_db.py

113 lines
3.3 KiB
Python

import io
import json
import os
import sqlite3
import sys
import requests
KAIKKI_TURKISH_URL = "https://kaikki.org/dictionary/Turkish/kaikki.org-dictionary-Turkish.jsonl"
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_DIR = os.path.join(BASE_DIR, "data")
DB_PATH = os.path.join(DATA_DIR, "lex.db")
def download_and_ingest_turkish():
print(f"Downloading Turkish lexicon from {KAIKKI_TURKISH_URL}...")
response = requests.get(KAIKKI_TURKISH_URL, stream=True)
response.raise_for_status()
conn = sqlite3.connect(DB_PATH)
cur = conn.cursor()
# Create tr_entries table
cur.execute("DROP TABLE IF EXISTS tr_entries;")
cur.execute("""
CREATE TABLE tr_entries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
word TEXT NOT NULL,
pos TEXT NOT NULL,
glosses TEXT NOT NULL,
examples TEXT NOT NULL,
synonyms TEXT NOT NULL
);
""")
cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_word ON tr_entries (word);")
cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_pos ON tr_entries (pos);")
insert_sql = """
INSERT INTO tr_entries (word, pos, glosses, examples, synonyms)
VALUES (?, ?, ?, ?, ?);
"""
batch = []
total_count = 0
print("Parsing and inserting Turkish entries into SQLite...")
for line in response.iter_lines():
if not line:
continue
try:
item = json.loads(line.decode("utf-8"))
except Exception:
continue
word = item.get("word", "").strip().lower()
pos = item.get("pos", "").strip().lower()
if not word or not pos:
continue
senses = item.get("senses", [])
glosses = []
examples = []
synonyms = []
for s in senses:
for g in s.get("glosses", []):
g_clean = g.strip()
if g_clean and g_clean not in glosses:
glosses.append(g_clean)
for ex in s.get("examples", []):
text = ex.get("text", "").strip()
eng_trans = ex.get("english", "").strip()
if text:
if eng_trans:
examples.append(f"{text} -> {eng_trans}")
else:
examples.append(text)
for syn in s.get("synonyms", []):
syn_word = syn.get("word", "").strip()
if syn_word and syn_word not in synonyms:
synonyms.append(syn_word)
if not glosses:
continue
batch.append((
word,
pos,
json.dumps(glosses, ensure_ascii=False),
json.dumps(examples, ensure_ascii=False),
json.dumps(synonyms, ensure_ascii=False),
))
total_count += 1
if len(batch) >= 5000:
cur.executemany(insert_sql, batch)
conn.commit()
batch = []
print(f"Inserted {total_count} Turkish entries...")
if batch:
cur.executemany(insert_sql, batch)
conn.commit()
conn.close()
print(f"Turkish lexicon ingestion complete. Total entries: {total_count}")
if __name__ == "__main__":
download_and_ingest_turkish()