import io import json import os import sqlite3 import sys import requests KAIKKI_TURKISH_URL = "https://kaikki.org/dictionary/Turkish/kaikki.org-dictionary-Turkish.jsonl" BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DATA_DIR = os.path.join(BASE_DIR, "data") DB_PATH = os.path.join(DATA_DIR, "lex.db") def download_and_ingest_turkish(): print(f"Downloading Turkish lexicon from {KAIKKI_TURKISH_URL}...") response = requests.get(KAIKKI_TURKISH_URL, stream=True) response.raise_for_status() conn = sqlite3.connect(DB_PATH) cur = conn.cursor() # Create tr_entries table cur.execute("DROP TABLE IF EXISTS tr_entries;") cur.execute(""" CREATE TABLE tr_entries ( id INTEGER PRIMARY KEY AUTOINCREMENT, word TEXT NOT NULL, pos TEXT NOT NULL, glosses TEXT NOT NULL, examples TEXT NOT NULL, synonyms TEXT NOT NULL ); """) cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_word ON tr_entries (word);") cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_pos ON tr_entries (pos);") insert_sql = """ INSERT INTO tr_entries (word, pos, glosses, examples, synonyms) VALUES (?, ?, ?, ?, ?); """ batch = [] total_count = 0 print("Parsing and inserting Turkish entries into SQLite...") for line in response.iter_lines(): if not line: continue try: item = json.loads(line.decode("utf-8")) except Exception: continue word = item.get("word", "").strip().lower() pos = item.get("pos", "").strip().lower() if not word or not pos: continue senses = item.get("senses", []) glosses = [] examples = [] synonyms = [] for s in senses: for g in s.get("glosses", []): g_clean = g.strip() if g_clean and g_clean not in glosses: glosses.append(g_clean) for ex in s.get("examples", []): text = ex.get("text", "").strip() eng_trans = ex.get("english", "").strip() if text: if eng_trans: examples.append(f"{text} -> {eng_trans}") else: examples.append(text) for syn in s.get("synonyms", []): syn_word = syn.get("word", "").strip() if syn_word and syn_word not in synonyms: synonyms.append(syn_word) if not glosses: continue batch.append(( word, pos, json.dumps(glosses, ensure_ascii=False), json.dumps(examples, ensure_ascii=False), json.dumps(synonyms, ensure_ascii=False), )) total_count += 1 if len(batch) >= 5000: cur.executemany(insert_sql, batch) conn.commit() batch = [] print(f"Inserted {total_count} Turkish entries...") if batch: cur.executemany(insert_sql, batch) conn.commit() conn.close() print(f"Turkish lexicon ingestion complete. Total entries: {total_count}") if __name__ == "__main__": download_and_ingest_turkish()