feat(lex): initial release v1.1.0 - offline bilingual dictionary engine
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import requests
|
||||
|
||||
KAIKKI_TURKISH_URL = "https://kaikki.org/dictionary/Turkish/kaikki.org-dictionary-Turkish.jsonl"
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DATA_DIR = os.path.join(BASE_DIR, "data")
|
||||
DB_PATH = os.path.join(DATA_DIR, "lex.db")
|
||||
|
||||
|
||||
def download_and_ingest_turkish():
|
||||
print(f"Downloading Turkish lexicon from {KAIKKI_TURKISH_URL}...")
|
||||
response = requests.get(KAIKKI_TURKISH_URL, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Create tr_entries table
|
||||
cur.execute("DROP TABLE IF EXISTS tr_entries;")
|
||||
cur.execute("""
|
||||
CREATE TABLE tr_entries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word TEXT NOT NULL,
|
||||
pos TEXT NOT NULL,
|
||||
glosses TEXT NOT NULL,
|
||||
examples TEXT NOT NULL,
|
||||
synonyms TEXT NOT NULL
|
||||
);
|
||||
""")
|
||||
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_word ON tr_entries (word);")
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_pos ON tr_entries (pos);")
|
||||
|
||||
insert_sql = """
|
||||
INSERT INTO tr_entries (word, pos, glosses, examples, synonyms)
|
||||
VALUES (?, ?, ?, ?, ?);
|
||||
"""
|
||||
|
||||
batch = []
|
||||
total_count = 0
|
||||
|
||||
print("Parsing and inserting Turkish entries into SQLite...")
|
||||
for line in response.iter_lines():
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line.decode("utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
word = item.get("word", "").strip().lower()
|
||||
pos = item.get("pos", "").strip().lower()
|
||||
if not word or not pos:
|
||||
continue
|
||||
|
||||
senses = item.get("senses", [])
|
||||
glosses = []
|
||||
examples = []
|
||||
synonyms = []
|
||||
|
||||
for s in senses:
|
||||
for g in s.get("glosses", []):
|
||||
g_clean = g.strip()
|
||||
if g_clean and g_clean not in glosses:
|
||||
glosses.append(g_clean)
|
||||
|
||||
for ex in s.get("examples", []):
|
||||
text = ex.get("text", "").strip()
|
||||
eng_trans = ex.get("english", "").strip()
|
||||
if text:
|
||||
if eng_trans:
|
||||
examples.append(f"{text} -> {eng_trans}")
|
||||
else:
|
||||
examples.append(text)
|
||||
|
||||
for syn in s.get("synonyms", []):
|
||||
syn_word = syn.get("word", "").strip()
|
||||
if syn_word and syn_word not in synonyms:
|
||||
synonyms.append(syn_word)
|
||||
|
||||
if not glosses:
|
||||
continue
|
||||
|
||||
batch.append((
|
||||
word,
|
||||
pos,
|
||||
json.dumps(glosses, ensure_ascii=False),
|
||||
json.dumps(examples, ensure_ascii=False),
|
||||
json.dumps(synonyms, ensure_ascii=False),
|
||||
))
|
||||
total_count += 1
|
||||
|
||||
if len(batch) >= 5000:
|
||||
cur.executemany(insert_sql, batch)
|
||||
conn.commit()
|
||||
batch = []
|
||||
print(f"Inserted {total_count} Turkish entries...")
|
||||
|
||||
if batch:
|
||||
cur.executemany(insert_sql, batch)
|
||||
conn.commit()
|
||||
|
||||
conn.close()
|
||||
print(f"Turkish lexicon ingestion complete. Total entries: {total_count}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
download_and_ingest_turkish()
|
||||
Reference in New Issue
Block a user