167 lines
5.1 KiB
Python
167 lines
5.1 KiB
Python
import io
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import zipfile
|
|
import requests
|
|
|
|
WORDNET_ZIP_URL = "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/wordnet.zip"
|
|
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
DATA_DIR = os.path.join(BASE_DIR, "data")
|
|
DB_PATH = os.path.join(DATA_DIR, "lex.db")
|
|
|
|
POS_MAP = {
|
|
"n": "noun",
|
|
"v": "verb",
|
|
"a": "adjective",
|
|
"s": "adjective satellite",
|
|
"r": "adverb"
|
|
}
|
|
|
|
def ensure_directories():
|
|
os.makedirs(DATA_DIR, exist_ok=True)
|
|
|
|
def download_wordnet():
|
|
print(f"Downloading WordNet from {WORDNET_ZIP_URL}...")
|
|
response = requests.get(WORDNET_ZIP_URL, stream=True)
|
|
response.raise_for_status()
|
|
total_size = int(response.headers.get("content-length", 0))
|
|
print(f"Downloaded package size: {total_size / (1024 * 1024):.2f} MB")
|
|
return io.BytesIO(response.content)
|
|
|
|
def parse_wordnet_data(zip_file):
|
|
records = []
|
|
with zipfile.ZipFile(zip_file) as z:
|
|
data_files = [
|
|
("wordnet/data.noun", "n"),
|
|
("wordnet/data.verb", "v"),
|
|
("wordnet/data.adj", "a"),
|
|
("wordnet/data.adv", "r"),
|
|
]
|
|
|
|
for zip_path, pos_code in data_files:
|
|
if zip_path not in z.namelist():
|
|
continue
|
|
with z.open(zip_path) as f:
|
|
for line_bytes in f:
|
|
line = line_bytes.decode("utf-8", errors="ignore").strip()
|
|
if not line or line.startswith(" "):
|
|
continue
|
|
|
|
parts = line.split("|")
|
|
if len(parts) < 2:
|
|
continue
|
|
|
|
gloss = parts[1].strip()
|
|
meta_part = parts[0].strip().split()
|
|
if len(meta_part) < 5:
|
|
continue
|
|
|
|
synset_offset = meta_part[0]
|
|
pos = POS_MAP.get(pos_code, pos_code)
|
|
|
|
# Extract lemmas
|
|
try:
|
|
w_cnt = int(meta_part[3], 16)
|
|
except ValueError:
|
|
continue
|
|
|
|
lemmas = []
|
|
idx = 4
|
|
for _ in range(w_cnt):
|
|
if idx < len(meta_part):
|
|
lemmas.append(meta_part[idx].lower().replace("_", " "))
|
|
idx += 2
|
|
|
|
# Split definition and examples from gloss
|
|
# Gloss format: "definition ; \"example 1\" ; \"example 2\""
|
|
gloss_split = gloss.split(";")
|
|
definition = gloss_split[0].strip()
|
|
examples = []
|
|
for s in gloss_split[1:]:
|
|
s_clean = s.strip().strip('"').strip()
|
|
if s_clean:
|
|
examples.append(s_clean)
|
|
|
|
for word in lemmas:
|
|
synonyms = [w for w in lemmas if w != word]
|
|
records.append({
|
|
"word": word,
|
|
"pos": pos,
|
|
"definition": definition,
|
|
"examples": examples,
|
|
"synonyms": synonyms,
|
|
"synset_id": f"{pos_code}.{synset_offset}"
|
|
})
|
|
|
|
return records
|
|
|
|
def create_sqlite_database(records):
|
|
print(f"Creating SQLite database at {DB_PATH} with {len(records)} entries...")
|
|
if os.path.exists(DB_PATH):
|
|
os.remove(DB_PATH)
|
|
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cur = conn.cursor()
|
|
|
|
# Create tables
|
|
cur.execute("""
|
|
CREATE TABLE entries (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
word TEXT NOT NULL,
|
|
pos TEXT NOT NULL,
|
|
definition TEXT NOT NULL,
|
|
examples TEXT NOT NULL,
|
|
synonyms TEXT NOT NULL,
|
|
synset_id TEXT NOT NULL
|
|
);
|
|
""")
|
|
|
|
cur.execute("CREATE INDEX idx_entries_word ON entries (word);")
|
|
cur.execute("CREATE INDEX idx_entries_pos ON entries (pos);")
|
|
|
|
# FTS5 table for fast word lookups
|
|
cur.execute("""
|
|
CREATE VIRTUAL TABLE entries_fts USING fts5(
|
|
word,
|
|
tokenize='unicode61 remove_diacritics 2'
|
|
);
|
|
""")
|
|
|
|
# Batch insert
|
|
insert_sql = """
|
|
INSERT INTO entries (word, pos, definition, examples, synonyms, synset_id)
|
|
VALUES (?, ?, ?, ?, ?, ?);
|
|
"""
|
|
fts_sql = "INSERT INTO entries_fts (word) VALUES (?);"
|
|
|
|
batch = []
|
|
fts_batch = []
|
|
for r in records:
|
|
batch.append((
|
|
r["word"],
|
|
r["pos"],
|
|
r["definition"],
|
|
json.dumps(r["examples"], ensure_ascii=False),
|
|
json.dumps(r["synonyms"], ensure_ascii=False),
|
|
r["synset_id"]
|
|
))
|
|
fts_batch.append((r["word"],))
|
|
|
|
cur.executemany(insert_sql, batch)
|
|
cur.executemany(fts_sql, fts_batch)
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
print("Database creation complete.")
|
|
|
|
def main():
|
|
ensure_directories()
|
|
zip_bytes = download_wordnet()
|
|
records = parse_wordnet_data(zip_bytes)
|
|
create_sqlite_database(records)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|