Files
lex/scripts/build_wordnet_db.py

167 lines
5.1 KiB
Python

import io
import json
import os
import sqlite3
import sys
import zipfile
import requests
WORDNET_ZIP_URL = "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/wordnet.zip"
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_DIR = os.path.join(BASE_DIR, "data")
DB_PATH = os.path.join(DATA_DIR, "lex.db")
POS_MAP = {
"n": "noun",
"v": "verb",
"a": "adjective",
"s": "adjective satellite",
"r": "adverb"
}
def ensure_directories():
os.makedirs(DATA_DIR, exist_ok=True)
def download_wordnet():
print(f"Downloading WordNet from {WORDNET_ZIP_URL}...")
response = requests.get(WORDNET_ZIP_URL, stream=True)
response.raise_for_status()
total_size = int(response.headers.get("content-length", 0))
print(f"Downloaded package size: {total_size / (1024 * 1024):.2f} MB")
return io.BytesIO(response.content)
def parse_wordnet_data(zip_file):
records = []
with zipfile.ZipFile(zip_file) as z:
data_files = [
("wordnet/data.noun", "n"),
("wordnet/data.verb", "v"),
("wordnet/data.adj", "a"),
("wordnet/data.adv", "r"),
]
for zip_path, pos_code in data_files:
if zip_path not in z.namelist():
continue
with z.open(zip_path) as f:
for line_bytes in f:
line = line_bytes.decode("utf-8", errors="ignore").strip()
if not line or line.startswith(" "):
continue
parts = line.split("|")
if len(parts) < 2:
continue
gloss = parts[1].strip()
meta_part = parts[0].strip().split()
if len(meta_part) < 5:
continue
synset_offset = meta_part[0]
pos = POS_MAP.get(pos_code, pos_code)
# Extract lemmas
try:
w_cnt = int(meta_part[3], 16)
except ValueError:
continue
lemmas = []
idx = 4
for _ in range(w_cnt):
if idx < len(meta_part):
lemmas.append(meta_part[idx].lower().replace("_", " "))
idx += 2
# Split definition and examples from gloss
# Gloss format: "definition ; \"example 1\" ; \"example 2\""
gloss_split = gloss.split(";")
definition = gloss_split[0].strip()
examples = []
for s in gloss_split[1:]:
s_clean = s.strip().strip('"').strip()
if s_clean:
examples.append(s_clean)
for word in lemmas:
synonyms = [w for w in lemmas if w != word]
records.append({
"word": word,
"pos": pos,
"definition": definition,
"examples": examples,
"synonyms": synonyms,
"synset_id": f"{pos_code}.{synset_offset}"
})
return records
def create_sqlite_database(records):
print(f"Creating SQLite database at {DB_PATH} with {len(records)} entries...")
if os.path.exists(DB_PATH):
os.remove(DB_PATH)
conn = sqlite3.connect(DB_PATH)
cur = conn.cursor()
# Create tables
cur.execute("""
CREATE TABLE entries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
word TEXT NOT NULL,
pos TEXT NOT NULL,
definition TEXT NOT NULL,
examples TEXT NOT NULL,
synonyms TEXT NOT NULL,
synset_id TEXT NOT NULL
);
""")
cur.execute("CREATE INDEX idx_entries_word ON entries (word);")
cur.execute("CREATE INDEX idx_entries_pos ON entries (pos);")
# FTS5 table for fast word lookups
cur.execute("""
CREATE VIRTUAL TABLE entries_fts USING fts5(
word,
tokenize='unicode61 remove_diacritics 2'
);
""")
# Batch insert
insert_sql = """
INSERT INTO entries (word, pos, definition, examples, synonyms, synset_id)
VALUES (?, ?, ?, ?, ?, ?);
"""
fts_sql = "INSERT INTO entries_fts (word) VALUES (?);"
batch = []
fts_batch = []
for r in records:
batch.append((
r["word"],
r["pos"],
r["definition"],
json.dumps(r["examples"], ensure_ascii=False),
json.dumps(r["synonyms"], ensure_ascii=False),
r["synset_id"]
))
fts_batch.append((r["word"],))
cur.executemany(insert_sql, batch)
cur.executemany(fts_sql, fts_batch)
conn.commit()
conn.close()
print("Database creation complete.")
def main():
ensure_directories()
zip_bytes = download_wordnet()
records = parse_wordnet_data(zip_bytes)
create_sqlite_database(records)
if __name__ == "__main__":
main()