import io import json import os import sqlite3 import sys import zipfile import requests WORDNET_ZIP_URL = "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/wordnet.zip" BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DATA_DIR = os.path.join(BASE_DIR, "data") DB_PATH = os.path.join(DATA_DIR, "lex.db") POS_MAP = { "n": "noun", "v": "verb", "a": "adjective", "s": "adjective satellite", "r": "adverb" } def ensure_directories(): os.makedirs(DATA_DIR, exist_ok=True) def download_wordnet(): print(f"Downloading WordNet from {WORDNET_ZIP_URL}...") response = requests.get(WORDNET_ZIP_URL, stream=True) response.raise_for_status() total_size = int(response.headers.get("content-length", 0)) print(f"Downloaded package size: {total_size / (1024 * 1024):.2f} MB") return io.BytesIO(response.content) def parse_wordnet_data(zip_file): records = [] with zipfile.ZipFile(zip_file) as z: data_files = [ ("wordnet/data.noun", "n"), ("wordnet/data.verb", "v"), ("wordnet/data.adj", "a"), ("wordnet/data.adv", "r"), ] for zip_path, pos_code in data_files: if zip_path not in z.namelist(): continue with z.open(zip_path) as f: for line_bytes in f: line = line_bytes.decode("utf-8", errors="ignore").strip() if not line or line.startswith(" "): continue parts = line.split("|") if len(parts) < 2: continue gloss = parts[1].strip() meta_part = parts[0].strip().split() if len(meta_part) < 5: continue synset_offset = meta_part[0] pos = POS_MAP.get(pos_code, pos_code) # Extract lemmas try: w_cnt = int(meta_part[3], 16) except ValueError: continue lemmas = [] idx = 4 for _ in range(w_cnt): if idx < len(meta_part): lemmas.append(meta_part[idx].lower().replace("_", " ")) idx += 2 # Split definition and examples from gloss # Gloss format: "definition ; \"example 1\" ; \"example 2\"" gloss_split = gloss.split(";") definition = gloss_split[0].strip() examples = [] for s in gloss_split[1:]: s_clean = s.strip().strip('"').strip() if s_clean: examples.append(s_clean) for word in lemmas: synonyms = [w for w in lemmas if w != word] records.append({ "word": word, "pos": pos, "definition": definition, "examples": examples, "synonyms": synonyms, "synset_id": f"{pos_code}.{synset_offset}" }) return records def create_sqlite_database(records): print(f"Creating SQLite database at {DB_PATH} with {len(records)} entries...") if os.path.exists(DB_PATH): os.remove(DB_PATH) conn = sqlite3.connect(DB_PATH) cur = conn.cursor() # Create tables cur.execute(""" CREATE TABLE entries ( id INTEGER PRIMARY KEY AUTOINCREMENT, word TEXT NOT NULL, pos TEXT NOT NULL, definition TEXT NOT NULL, examples TEXT NOT NULL, synonyms TEXT NOT NULL, synset_id TEXT NOT NULL ); """) cur.execute("CREATE INDEX idx_entries_word ON entries (word);") cur.execute("CREATE INDEX idx_entries_pos ON entries (pos);") # FTS5 table for fast word lookups cur.execute(""" CREATE VIRTUAL TABLE entries_fts USING fts5( word, tokenize='unicode61 remove_diacritics 2' ); """) # Batch insert insert_sql = """ INSERT INTO entries (word, pos, definition, examples, synonyms, synset_id) VALUES (?, ?, ?, ?, ?, ?); """ fts_sql = "INSERT INTO entries_fts (word) VALUES (?);" batch = [] fts_batch = [] for r in records: batch.append(( r["word"], r["pos"], r["definition"], json.dumps(r["examples"], ensure_ascii=False), json.dumps(r["synonyms"], ensure_ascii=False), r["synset_id"] )) fts_batch.append((r["word"],)) cur.executemany(insert_sql, batch) cur.executemany(fts_sql, fts_batch) conn.commit() conn.close() print("Database creation complete.") def main(): ensure_directories() zip_bytes = download_wordnet() records = parse_wordnet_data(zip_bytes) create_sqlite_database(records) if __name__ == "__main__": main()