feat(lex): initial release v1.1.0 - offline bilingual dictionary engine
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import requests
|
||||
|
||||
KAIKKI_TURKISH_URL = "https://kaikki.org/dictionary/Turkish/kaikki.org-dictionary-Turkish.jsonl"
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DATA_DIR = os.path.join(BASE_DIR, "data")
|
||||
DB_PATH = os.path.join(DATA_DIR, "lex.db")
|
||||
|
||||
|
||||
def download_and_ingest_turkish():
|
||||
print(f"Downloading Turkish lexicon from {KAIKKI_TURKISH_URL}...")
|
||||
response = requests.get(KAIKKI_TURKISH_URL, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Create tr_entries table
|
||||
cur.execute("DROP TABLE IF EXISTS tr_entries;")
|
||||
cur.execute("""
|
||||
CREATE TABLE tr_entries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word TEXT NOT NULL,
|
||||
pos TEXT NOT NULL,
|
||||
glosses TEXT NOT NULL,
|
||||
examples TEXT NOT NULL,
|
||||
synonyms TEXT NOT NULL
|
||||
);
|
||||
""")
|
||||
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_word ON tr_entries (word);")
|
||||
cur.execute("CREATE INDEX IF NOT EXISTS idx_tr_entries_pos ON tr_entries (pos);")
|
||||
|
||||
insert_sql = """
|
||||
INSERT INTO tr_entries (word, pos, glosses, examples, synonyms)
|
||||
VALUES (?, ?, ?, ?, ?);
|
||||
"""
|
||||
|
||||
batch = []
|
||||
total_count = 0
|
||||
|
||||
print("Parsing and inserting Turkish entries into SQLite...")
|
||||
for line in response.iter_lines():
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line.decode("utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
word = item.get("word", "").strip().lower()
|
||||
pos = item.get("pos", "").strip().lower()
|
||||
if not word or not pos:
|
||||
continue
|
||||
|
||||
senses = item.get("senses", [])
|
||||
glosses = []
|
||||
examples = []
|
||||
synonyms = []
|
||||
|
||||
for s in senses:
|
||||
for g in s.get("glosses", []):
|
||||
g_clean = g.strip()
|
||||
if g_clean and g_clean not in glosses:
|
||||
glosses.append(g_clean)
|
||||
|
||||
for ex in s.get("examples", []):
|
||||
text = ex.get("text", "").strip()
|
||||
eng_trans = ex.get("english", "").strip()
|
||||
if text:
|
||||
if eng_trans:
|
||||
examples.append(f"{text} -> {eng_trans}")
|
||||
else:
|
||||
examples.append(text)
|
||||
|
||||
for syn in s.get("synonyms", []):
|
||||
syn_word = syn.get("word", "").strip()
|
||||
if syn_word and syn_word not in synonyms:
|
||||
synonyms.append(syn_word)
|
||||
|
||||
if not glosses:
|
||||
continue
|
||||
|
||||
batch.append((
|
||||
word,
|
||||
pos,
|
||||
json.dumps(glosses, ensure_ascii=False),
|
||||
json.dumps(examples, ensure_ascii=False),
|
||||
json.dumps(synonyms, ensure_ascii=False),
|
||||
))
|
||||
total_count += 1
|
||||
|
||||
if len(batch) >= 5000:
|
||||
cur.executemany(insert_sql, batch)
|
||||
conn.commit()
|
||||
batch = []
|
||||
print(f"Inserted {total_count} Turkish entries...")
|
||||
|
||||
if batch:
|
||||
cur.executemany(insert_sql, batch)
|
||||
conn.commit()
|
||||
|
||||
conn.close()
|
||||
print(f"Turkish lexicon ingestion complete. Total entries: {total_count}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
download_and_ingest_turkish()
|
||||
@@ -0,0 +1,166 @@
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import zipfile
|
||||
import requests
|
||||
|
||||
WORDNET_ZIP_URL = "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/wordnet.zip"
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DATA_DIR = os.path.join(BASE_DIR, "data")
|
||||
DB_PATH = os.path.join(DATA_DIR, "lex.db")
|
||||
|
||||
POS_MAP = {
|
||||
"n": "noun",
|
||||
"v": "verb",
|
||||
"a": "adjective",
|
||||
"s": "adjective satellite",
|
||||
"r": "adverb"
|
||||
}
|
||||
|
||||
def ensure_directories():
|
||||
os.makedirs(DATA_DIR, exist_ok=True)
|
||||
|
||||
def download_wordnet():
|
||||
print(f"Downloading WordNet from {WORDNET_ZIP_URL}...")
|
||||
response = requests.get(WORDNET_ZIP_URL, stream=True)
|
||||
response.raise_for_status()
|
||||
total_size = int(response.headers.get("content-length", 0))
|
||||
print(f"Downloaded package size: {total_size / (1024 * 1024):.2f} MB")
|
||||
return io.BytesIO(response.content)
|
||||
|
||||
def parse_wordnet_data(zip_file):
|
||||
records = []
|
||||
with zipfile.ZipFile(zip_file) as z:
|
||||
data_files = [
|
||||
("wordnet/data.noun", "n"),
|
||||
("wordnet/data.verb", "v"),
|
||||
("wordnet/data.adj", "a"),
|
||||
("wordnet/data.adv", "r"),
|
||||
]
|
||||
|
||||
for zip_path, pos_code in data_files:
|
||||
if zip_path not in z.namelist():
|
||||
continue
|
||||
with z.open(zip_path) as f:
|
||||
for line_bytes in f:
|
||||
line = line_bytes.decode("utf-8", errors="ignore").strip()
|
||||
if not line or line.startswith(" "):
|
||||
continue
|
||||
|
||||
parts = line.split("|")
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
|
||||
gloss = parts[1].strip()
|
||||
meta_part = parts[0].strip().split()
|
||||
if len(meta_part) < 5:
|
||||
continue
|
||||
|
||||
synset_offset = meta_part[0]
|
||||
pos = POS_MAP.get(pos_code, pos_code)
|
||||
|
||||
# Extract lemmas
|
||||
try:
|
||||
w_cnt = int(meta_part[3], 16)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
lemmas = []
|
||||
idx = 4
|
||||
for _ in range(w_cnt):
|
||||
if idx < len(meta_part):
|
||||
lemmas.append(meta_part[idx].lower().replace("_", " "))
|
||||
idx += 2
|
||||
|
||||
# Split definition and examples from gloss
|
||||
# Gloss format: "definition ; \"example 1\" ; \"example 2\""
|
||||
gloss_split = gloss.split(";")
|
||||
definition = gloss_split[0].strip()
|
||||
examples = []
|
||||
for s in gloss_split[1:]:
|
||||
s_clean = s.strip().strip('"').strip()
|
||||
if s_clean:
|
||||
examples.append(s_clean)
|
||||
|
||||
for word in lemmas:
|
||||
synonyms = [w for w in lemmas if w != word]
|
||||
records.append({
|
||||
"word": word,
|
||||
"pos": pos,
|
||||
"definition": definition,
|
||||
"examples": examples,
|
||||
"synonyms": synonyms,
|
||||
"synset_id": f"{pos_code}.{synset_offset}"
|
||||
})
|
||||
|
||||
return records
|
||||
|
||||
def create_sqlite_database(records):
|
||||
print(f"Creating SQLite database at {DB_PATH} with {len(records)} entries...")
|
||||
if os.path.exists(DB_PATH):
|
||||
os.remove(DB_PATH)
|
||||
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Create tables
|
||||
cur.execute("""
|
||||
CREATE TABLE entries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word TEXT NOT NULL,
|
||||
pos TEXT NOT NULL,
|
||||
definition TEXT NOT NULL,
|
||||
examples TEXT NOT NULL,
|
||||
synonyms TEXT NOT NULL,
|
||||
synset_id TEXT NOT NULL
|
||||
);
|
||||
""")
|
||||
|
||||
cur.execute("CREATE INDEX idx_entries_word ON entries (word);")
|
||||
cur.execute("CREATE INDEX idx_entries_pos ON entries (pos);")
|
||||
|
||||
# FTS5 table for fast word lookups
|
||||
cur.execute("""
|
||||
CREATE VIRTUAL TABLE entries_fts USING fts5(
|
||||
word,
|
||||
tokenize='unicode61 remove_diacritics 2'
|
||||
);
|
||||
""")
|
||||
|
||||
# Batch insert
|
||||
insert_sql = """
|
||||
INSERT INTO entries (word, pos, definition, examples, synonyms, synset_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?);
|
||||
"""
|
||||
fts_sql = "INSERT INTO entries_fts (word) VALUES (?);"
|
||||
|
||||
batch = []
|
||||
fts_batch = []
|
||||
for r in records:
|
||||
batch.append((
|
||||
r["word"],
|
||||
r["pos"],
|
||||
r["definition"],
|
||||
json.dumps(r["examples"], ensure_ascii=False),
|
||||
json.dumps(r["synonyms"], ensure_ascii=False),
|
||||
r["synset_id"]
|
||||
))
|
||||
fts_batch.append((r["word"],))
|
||||
|
||||
cur.executemany(insert_sql, batch)
|
||||
cur.executemany(fts_sql, fts_batch)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print("Database creation complete.")
|
||||
|
||||
def main():
|
||||
ensure_directories()
|
||||
zip_bytes = download_wordnet()
|
||||
records = parse_wordnet_data(zip_bytes)
|
||||
create_sqlite_database(records)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
ASSETS_DIR = BASE_DIR / "assets"
|
||||
ASSETS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
ICON_PATH = ASSETS_DIR / "lex_icon.ico"
|
||||
|
||||
|
||||
def generate_icon():
|
||||
size = 256
|
||||
img = Image.new("RGBA", (size, size), (0, 0, 0, 0))
|
||||
draw = ImageDraw.Draw(img)
|
||||
|
||||
# Draw rounded rectangle badge background
|
||||
pad = 16
|
||||
draw.rounded_rectangle(
|
||||
[pad, pad, size - pad, size - pad],
|
||||
radius=48,
|
||||
fill=(24, 119, 242, 255), # Modern royal blue
|
||||
outline=(200, 230, 255, 255),
|
||||
width=4,
|
||||
)
|
||||
|
||||
# Draw inner terminal / book symbol
|
||||
# Book left page
|
||||
draw.polygon(
|
||||
[(64, 80), (124, 70), (124, 180), (64, 170)],
|
||||
fill=(255, 255, 255, 240),
|
||||
)
|
||||
# Book right page
|
||||
draw.polygon(
|
||||
[(132, 70), (192, 80), (192, 170), (132, 180)],
|
||||
fill=(255, 255, 255, 240),
|
||||
)
|
||||
|
||||
# Spine separator line
|
||||
draw.line([(128, 68), (128, 182)], fill=(20, 80, 180, 255), width=3)
|
||||
|
||||
# Text / lines on pages
|
||||
draw.line([(76, 100), (114, 94)], fill=(100, 150, 220, 255), width=3)
|
||||
draw.line([(76, 120), (114, 114)], fill=(100, 150, 220, 255), width=3)
|
||||
draw.line([(76, 140), (114, 134)], fill=(100, 150, 220, 255), width=3)
|
||||
|
||||
draw.line([(142, 94), (180, 100)], fill=(100, 150, 220, 255), width=3)
|
||||
draw.line([(142, 114), (180, 120)], fill=(100, 150, 220, 255), width=3)
|
||||
draw.line([(142, 134), (180, 140)], fill=(100, 150, 220, 255), width=3)
|
||||
|
||||
# Save as multi-size ICO
|
||||
sizes = [(16, 16), (32, 32), (48, 48), (64, 64), (128, 128), (256, 256)]
|
||||
img.save(ICON_PATH, format="ICO", sizes=sizes)
|
||||
print(f"Generated icon at: {ICON_PATH}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate_icon()
|
||||
Reference in New Issue
Block a user