feat(lex): initial release v1.1.0 - offline bilingual dictionary engine

This commit is contained in:
2026-10-03 03:43:42 +03:00
commit dc9b2e7fe4
29 changed files with 1738 additions and 0 deletions
+112
View File
@@ -0,0 +1,112 @@
import io
import json
import os
import sqlite3
import sys
import requests
KAIKKI_TURKISH_URL = "https://kaikki.org/dictionary/Turkish/kaikki.org-dictionary-Turkish.jsonl"
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_DIR = os.path.join(BASE_DIR, "data")
DB_PATH = os.path.join(DATA_DIR, "lex.db")
def download_and_ingest_turkish():
print(f"Downloading Turkish lexicon from {KAIKKI_TURKISH_URL}...")
response = requests.get(KAIKKI_TURKISH_URL, stream=True)
response.raise_for_status()
conn = sqlite3.connect(DB_PATH)
cur = conn.cursor()
# Create tr_entries table
cur.execute("DROP TABLE IF EXISTS tr_entries;")
cur.execute("""
CREATE TABLE tr_entries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
word TEXT NOT NULL,
pos TEXT NOT NULL,
glosses TEXT NOT NULL,
examples TEXT NOT NULL,
synonyms TEXT NOT NULL
);
""")
cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_word ON tr_entries (word);")
cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_pos ON tr_entries (pos);")
insert_sql = """
INSERT INTO tr_entries (word, pos, glosses, examples, synonyms)
VALUES (?, ?, ?, ?, ?);
"""
batch = []
total_count = 0
print("Parsing and inserting Turkish entries into SQLite...")
for line in response.iter_lines():
if not line:
continue
try:
item = json.loads(line.decode("utf-8"))
except Exception:
continue
word = item.get("word", "").strip().lower()
pos = item.get("pos", "").strip().lower()
if not word or not pos:
continue
senses = item.get("senses", [])
glosses = []
examples = []
synonyms = []
for s in senses:
for g in s.get("glosses", []):
g_clean = g.strip()
if g_clean and g_clean not in glosses:
glosses.append(g_clean)
for ex in s.get("examples", []):
text = ex.get("text", "").strip()
eng_trans = ex.get("english", "").strip()
if text:
if eng_trans:
examples.append(f"{text} -> {eng_trans}")
else:
examples.append(text)
for syn in s.get("synonyms", []):
syn_word = syn.get("word", "").strip()
if syn_word and syn_word not in synonyms:
synonyms.append(syn_word)
if not glosses:
continue
batch.append((
word,
pos,
json.dumps(glosses, ensure_ascii=False),
json.dumps(examples, ensure_ascii=False),
json.dumps(synonyms, ensure_ascii=False),
))
total_count += 1
if len(batch) >= 5000:
cur.executemany(insert_sql, batch)
conn.commit()
batch = []
print(f"Inserted {total_count} Turkish entries...")
if batch:
cur.executemany(insert_sql, batch)
conn.commit()
conn.close()
print(f"Turkish lexicon ingestion complete. Total entries: {total_count}")
if __name__ == "__main__":
download_and_ingest_turkish()
+166
View File
@@ -0,0 +1,166 @@
import io
import json
import os
import sqlite3
import sys
import zipfile
import requests
WORDNET_ZIP_URL = "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/wordnet.zip"
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_DIR = os.path.join(BASE_DIR, "data")
DB_PATH = os.path.join(DATA_DIR, "lex.db")
POS_MAP = {
"n": "noun",
"v": "verb",
"a": "adjective",
"s": "adjective satellite",
"r": "adverb"
}
def ensure_directories():
os.makedirs(DATA_DIR, exist_ok=True)
def download_wordnet():
print(f"Downloading WordNet from {WORDNET_ZIP_URL}...")
response = requests.get(WORDNET_ZIP_URL, stream=True)
response.raise_for_status()
total_size = int(response.headers.get("content-length", 0))
print(f"Downloaded package size: {total_size / (1024 * 1024):.2f} MB")
return io.BytesIO(response.content)
def parse_wordnet_data(zip_file):
records = []
with zipfile.ZipFile(zip_file) as z:
data_files = [
("wordnet/data.noun", "n"),
("wordnet/data.verb", "v"),
("wordnet/data.adj", "a"),
("wordnet/data.adv", "r"),
]
for zip_path, pos_code in data_files:
if zip_path not in z.namelist():
continue
with z.open(zip_path) as f:
for line_bytes in f:
line = line_bytes.decode("utf-8", errors="ignore").strip()
if not line or line.startswith(" "):
continue
parts = line.split("|")
if len(parts) < 2:
continue
gloss = parts[1].strip()
meta_part = parts[0].strip().split()
if len(meta_part) < 5:
continue
synset_offset = meta_part[0]
pos = POS_MAP.get(pos_code, pos_code)
# Extract lemmas
try:
w_cnt = int(meta_part[3], 16)
except ValueError:
continue
lemmas = []
idx = 4
for _ in range(w_cnt):
if idx < len(meta_part):
lemmas.append(meta_part[idx].lower().replace("_", " "))
idx += 2
# Split definition and examples from gloss
# Gloss format: "definition ; \"example 1\" ; \"example 2\""
gloss_split = gloss.split(";")
definition = gloss_split[0].strip()
examples = []
for s in gloss_split[1:]:
s_clean = s.strip().strip('"').strip()
if s_clean:
examples.append(s_clean)
for word in lemmas:
synonyms = [w for w in lemmas if w != word]
records.append({
"word": word,
"pos": pos,
"definition": definition,
"examples": examples,
"synonyms": synonyms,
"synset_id": f"{pos_code}.{synset_offset}"
})
return records
def create_sqlite_database(records):
print(f"Creating SQLite database at {DB_PATH} with {len(records)} entries...")
if os.path.exists(DB_PATH):
os.remove(DB_PATH)
conn = sqlite3.connect(DB_PATH)
cur = conn.cursor()
# Create tables
cur.execute("""
CREATE TABLE entries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
word TEXT NOT NULL,
pos TEXT NOT NULL,
definition TEXT NOT NULL,
examples TEXT NOT NULL,
synonyms TEXT NOT NULL,
synset_id TEXT NOT NULL
);
""")
cur.execute("CREATE INDEX idx_entries_word ON entries (word);")
cur.execute("CREATE INDEX idx_entries_pos ON entries (pos);")
# FTS5 table for fast word lookups
cur.execute("""
CREATE VIRTUAL TABLE entries_fts USING fts5(
word,
tokenize='unicode61 remove_diacritics 2'
);
""")
# Batch insert
insert_sql = """
INSERT INTO entries (word, pos, definition, examples, synonyms, synset_id)
VALUES (?, ?, ?, ?, ?, ?);
"""
fts_sql = "INSERT INTO entries_fts (word) VALUES (?);"
batch = []
fts_batch = []
for r in records:
batch.append((
r["word"],
r["pos"],
r["definition"],
json.dumps(r["examples"], ensure_ascii=False),
json.dumps(r["synonyms"], ensure_ascii=False),
r["synset_id"]
))
fts_batch.append((r["word"],))
cur.executemany(insert_sql, batch)
cur.executemany(fts_sql, fts_batch)
conn.commit()
conn.close()
print("Database creation complete.")
def main():
ensure_directories()
zip_bytes = download_wordnet()
records = parse_wordnet_data(zip_bytes)
create_sqlite_database(records)
if __name__ == "__main__":
main()
+57
View File
@@ -0,0 +1,57 @@
import os
from pathlib import Path
from PIL import Image, ImageDraw, ImageFont
BASE_DIR = Path(__file__).resolve().parent.parent
ASSETS_DIR = BASE_DIR / "assets"
ASSETS_DIR.mkdir(parents=True, exist_ok=True)
ICON_PATH = ASSETS_DIR / "lex_icon.ico"
def generate_icon():
size = 256
img = Image.new("RGBA", (size, size), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
# Draw rounded rectangle badge background
pad = 16
draw.rounded_rectangle(
[pad, pad, size - pad, size - pad],
radius=48,
fill=(24, 119, 242, 255), # Modern royal blue
outline=(200, 230, 255, 255),
width=4,
)
# Draw inner terminal / book symbol
# Book left page
draw.polygon(
[(64, 80), (124, 70), (124, 180), (64, 170)],
fill=(255, 255, 255, 240),
)
# Book right page
draw.polygon(
[(132, 70), (192, 80), (192, 170), (132, 180)],
fill=(255, 255, 255, 240),
)
# Spine separator line
draw.line([(128, 68), (128, 182)], fill=(20, 80, 180, 255), width=3)
# Text / lines on pages
draw.line([(76, 100), (114, 94)], fill=(100, 150, 220, 255), width=3)
draw.line([(76, 120), (114, 114)], fill=(100, 150, 220, 255), width=3)
draw.line([(76, 140), (114, 134)], fill=(100, 150, 220, 255), width=3)
draw.line([(142, 94), (180, 100)], fill=(100, 150, 220, 255), width=3)
draw.line([(142, 114), (180, 120)], fill=(100, 150, 220, 255), width=3)
draw.line([(142, 134), (180, 140)], fill=(100, 150, 220, 255), width=3)
# Save as multi-size ICO
sizes = [(16, 16), (32, 32), (48, 48), (64, 64), (128, 128), (256, 256)]
img.save(ICON_PATH, format="ICO", sizes=sizes)
print(f"Generated icon at: {ICON_PATH}")
if __name__ == "__main__":
generate_icon()