feat(lex): initial release v1.1.0 - offline bilingual dictionary engine
This commit is contained in:
@@ -0,0 +1,166 @@
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import zipfile
|
||||
import requests
|
||||
|
||||
WORDNET_ZIP_URL = "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/wordnet.zip"
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DATA_DIR = os.path.join(BASE_DIR, "data")
|
||||
DB_PATH = os.path.join(DATA_DIR, "lex.db")
|
||||
|
||||
POS_MAP = {
|
||||
"n": "noun",
|
||||
"v": "verb",
|
||||
"a": "adjective",
|
||||
"s": "adjective satellite",
|
||||
"r": "adverb"
|
||||
}
|
||||
|
||||
def ensure_directories():
|
||||
os.makedirs(DATA_DIR, exist_ok=True)
|
||||
|
||||
def download_wordnet():
|
||||
print(f"Downloading WordNet from {WORDNET_ZIP_URL}...")
|
||||
response = requests.get(WORDNET_ZIP_URL, stream=True)
|
||||
response.raise_for_status()
|
||||
total_size = int(response.headers.get("content-length", 0))
|
||||
print(f"Downloaded package size: {total_size / (1024 * 1024):.2f} MB")
|
||||
return io.BytesIO(response.content)
|
||||
|
||||
def parse_wordnet_data(zip_file):
|
||||
records = []
|
||||
with zipfile.ZipFile(zip_file) as z:
|
||||
data_files = [
|
||||
("wordnet/data.noun", "n"),
|
||||
("wordnet/data.verb", "v"),
|
||||
("wordnet/data.adj", "a"),
|
||||
("wordnet/data.adv", "r"),
|
||||
]
|
||||
|
||||
for zip_path, pos_code in data_files:
|
||||
if zip_path not in z.namelist():
|
||||
continue
|
||||
with z.open(zip_path) as f:
|
||||
for line_bytes in f:
|
||||
line = line_bytes.decode("utf-8", errors="ignore").strip()
|
||||
if not line or line.startswith(" "):
|
||||
continue
|
||||
|
||||
parts = line.split("|")
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
|
||||
gloss = parts[1].strip()
|
||||
meta_part = parts[0].strip().split()
|
||||
if len(meta_part) < 5:
|
||||
continue
|
||||
|
||||
synset_offset = meta_part[0]
|
||||
pos = POS_MAP.get(pos_code, pos_code)
|
||||
|
||||
# Extract lemmas
|
||||
try:
|
||||
w_cnt = int(meta_part[3], 16)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
lemmas = []
|
||||
idx = 4
|
||||
for _ in range(w_cnt):
|
||||
if idx < len(meta_part):
|
||||
lemmas.append(meta_part[idx].lower().replace("_", " "))
|
||||
idx += 2
|
||||
|
||||
# Split definition and examples from gloss
|
||||
# Gloss format: "definition ; \"example 1\" ; \"example 2\""
|
||||
gloss_split = gloss.split(";")
|
||||
definition = gloss_split[0].strip()
|
||||
examples = []
|
||||
for s in gloss_split[1:]:
|
||||
s_clean = s.strip().strip('"').strip()
|
||||
if s_clean:
|
||||
examples.append(s_clean)
|
||||
|
||||
for word in lemmas:
|
||||
synonyms = [w for w in lemmas if w != word]
|
||||
records.append({
|
||||
"word": word,
|
||||
"pos": pos,
|
||||
"definition": definition,
|
||||
"examples": examples,
|
||||
"synonyms": synonyms,
|
||||
"synset_id": f"{pos_code}.{synset_offset}"
|
||||
})
|
||||
|
||||
return records
|
||||
|
||||
def create_sqlite_database(records):
|
||||
print(f"Creating SQLite database at {DB_PATH} with {len(records)} entries...")
|
||||
if os.path.exists(DB_PATH):
|
||||
os.remove(DB_PATH)
|
||||
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Create tables
|
||||
cur.execute("""
|
||||
CREATE TABLE entries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word TEXT NOT NULL,
|
||||
pos TEXT NOT NULL,
|
||||
definition TEXT NOT NULL,
|
||||
examples TEXT NOT NULL,
|
||||
synonyms TEXT NOT NULL,
|
||||
synset_id TEXT NOT NULL
|
||||
);
|
||||
""")
|
||||
|
||||
cur.execute("CREATE INDEX idx_entries_word ON entries (word);")
|
||||
cur.execute("CREATE INDEX idx_entries_pos ON entries (pos);")
|
||||
|
||||
# FTS5 table for fast word lookups
|
||||
cur.execute("""
|
||||
CREATE VIRTUAL TABLE entries_fts USING fts5(
|
||||
word,
|
||||
tokenize='unicode61 remove_diacritics 2'
|
||||
);
|
||||
""")
|
||||
|
||||
# Batch insert
|
||||
insert_sql = """
|
||||
INSERT INTO entries (word, pos, definition, examples, synonyms, synset_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?);
|
||||
"""
|
||||
fts_sql = "INSERT INTO entries_fts (word) VALUES (?);"
|
||||
|
||||
batch = []
|
||||
fts_batch = []
|
||||
for r in records:
|
||||
batch.append((
|
||||
r["word"],
|
||||
r["pos"],
|
||||
r["definition"],
|
||||
json.dumps(r["examples"], ensure_ascii=False),
|
||||
json.dumps(r["synonyms"], ensure_ascii=False),
|
||||
r["synset_id"]
|
||||
))
|
||||
fts_batch.append((r["word"],))
|
||||
|
||||
cur.executemany(insert_sql, batch)
|
||||
cur.executemany(fts_sql, fts_batch)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print("Database creation complete.")
|
||||
|
||||
def main():
|
||||
ensure_directories()
|
||||
zip_bytes = download_wordnet()
|
||||
records = parse_wordnet_data(zip_bytes)
|
||||
create_sqlite_database(records)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user