357 lines
9.8 KiB
Markdown
357 lines
9.8 KiB
Markdown
---
|
|
created: 2026-08-06T15:15:00
|
|
tags:
|
|
- NotebookLM
|
|
- colab
|
|
- vector
|
|
updated: 2026-08-06T15:15:00
|
|
---
|
|
|
|
# REFACTORED COLAB NOTEBOOK - Free Tier Compatible
|
|
## Kentucky Kernel Archive → Vector DB Pipeline
|
|
|
|
---
|
|
|
|
## CELL 1: SETUP & MOUNT (DO THIS FIRST)
|
|
|
|
```python
|
|
## @title SETUP (RUN ONCE)
|
|
|
|
## Install everything upfront
|
|
!pip install internetarchive chromadb sentence-transformers
|
|
|
|
## Mount Drive once, reuse everywhere
|
|
from google.colab import drive
|
|
drive.mount('/content/drive')
|
|
|
|
print("✅ Setup complete. Drive mounted.")
|
|
```
|
|
|
|
---
|
|
|
|
## CELL 2: CONFIGURE PATHS (USER INPUT)
|
|
|
|
```python
|
|
## @title CONFIG - Enter Your Settings
|
|
|
|
## User inputs
|
|
creator_name = input("Enter creator name (e.g., 'The Mt. Sterling Advocate'): ").strip()
|
|
test_mode = input("Test mode? (y/n, tests with 50 issues): ").strip().lower() == 'y'
|
|
|
|
## Paths
|
|
BASE_DOWNLOAD = f'/content/{creator_name.replace(" ", "_")}_Archive'
|
|
CLEANED_JSON = f'/content/drive/MyDrive/{creator_name.replace(" ", "_")}_Cleaned/cleaned_pages.json'
|
|
VECTORDB_PATH = f'/content/drive/MyDrive/{creator_name.replace(" ", "_")}_VectorDB'
|
|
|
|
## Create directories
|
|
import os
|
|
os.makedirs(os.path.dirname(CLEANED_JSON), exist_ok=True)
|
|
os.makedirs(BASE_DOWNLOAD, exist_ok=True)
|
|
|
|
print(f"✓ Creator: {creator_name}")
|
|
print(f"✓ Download to: {BASE_DOWNLOAD}")
|
|
print(f"✓ Cleaned JSON: {CLEANED_JSON}")
|
|
print(f"✓ Vector DB: {VECTORDB_PATH}")
|
|
print(f"✓ Test Mode: {test_mode}")
|
|
```
|
|
|
|
---
|
|
|
|
## CELL 3: DOWNLOAD FROM ARCHIVE.ORG
|
|
|
|
```python
|
|
## @title DOWNLOAD (This takes 30-45 min)
|
|
|
|
from internetarchive import search_items, get_item, get_session
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
ia_session = get_session()
|
|
ia_session.mount_http_adapter()
|
|
|
|
## Search
|
|
query = f'creator:"{creator_name}"'
|
|
print(f"Searching: {query}")
|
|
|
|
search = search_items(query, archive_session=ia_session)
|
|
identifiers = [result['identifier'] for result in search]
|
|
|
|
if test_mode:
|
|
identifiers = identifiers[:50] # Test with 50 only
|
|
|
|
total_items = len(identifiers)
|
|
print(f"Found {total_items} issues. Starting download with 2 workers (safe)...\n")
|
|
|
|
def fast_download(identifier):
|
|
target_folder = os.path.join(BASE_DOWNLOAD, identifier)
|
|
if os.path.exists(target_folder) and len(os.listdir(target_folder)) >= 3:
|
|
return f"⏭️ Skipped: {identifier}"
|
|
|
|
try:
|
|
item = get_item(identifier, archive_session=ia_session)
|
|
item.download(
|
|
destdir=BASE_DOWNLOAD,
|
|
glob_pattern=['*meta.xml', '*djvu.xml', '*djvu.txt'],
|
|
ignore_existing=True,
|
|
retries=3
|
|
)
|
|
return f"✅ Downloaded: {identifier}"
|
|
except Exception as e:
|
|
return f"❌ Failed: {identifier} - {str(e)[:50]}"
|
|
|
|
## Use 2 workers (safer than 4 on Free Tier)
|
|
completed = 0
|
|
failed = []
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
futures = {executor.submit(fast_download, identifier): identifier for identifier in identifiers}
|
|
|
|
for future in as_completed(futures):
|
|
completed += 1
|
|
result = future.result()
|
|
if "❌" in result:
|
|
failed.append(result)
|
|
|
|
if completed % 25 == 0 or completed == total_items:
|
|
print(f"[{completed}/{total_items}] {result[:60]}")
|
|
|
|
print(f"\n✅ Download complete. {len(failed)} failures.")
|
|
if failed:
|
|
print("Failed items:")
|
|
for f in failed[:5]:
|
|
print(f" {f}")
|
|
```
|
|
|
|
---
|
|
|
|
## CELL 4: PARSE XML & CLEAN TEXT (CPU-BASED)
|
|
|
|
```python
|
|
## @title PARSE & CLEAN
|
|
|
|
import xml.etree.ElementTree as ET
|
|
import json
|
|
import pandas as pd # ← Use pandas, not cudf
|
|
import glob
|
|
|
|
print("Scanning for DJVU XML files...")
|
|
djvu_files = glob.glob(os.path.join(BASE_DOWNLOAD, '**/*djvu.xml'), recursive=True)
|
|
print(f"Found {len(djvu_files)} XML files.")
|
|
|
|
page_data_list = []
|
|
failed_issues = []
|
|
|
|
for djvu_path in djvu_files:
|
|
folder_name = os.path.basename(os.path.dirname(djvu_path))
|
|
|
|
try:
|
|
tree = ET.parse(djvu_path)
|
|
root = tree.getroot()
|
|
pages = root.findall('.//OBJECT')
|
|
|
|
for page_index, page in enumerate(pages):
|
|
page_text = []
|
|
for word in page.findall('.//WORD'):
|
|
if word.text:
|
|
page_text.append(word.text)
|
|
|
|
raw_page = " ".join(page_text)
|
|
|
|
if raw_page.strip():
|
|
page_data_list.append({
|
|
"issue_id": folder_name,
|
|
"page_number": page_index + 1,
|
|
"text": raw_page
|
|
})
|
|
|
|
except Exception as e:
|
|
failed_issues.append((folder_name, str(e)))
|
|
|
|
print(f"Extracted {len(page_data_list)} pages, {len(failed_issues)} failures.")
|
|
|
|
## CLEANING: Use pandas (CPU) - faster than GPU for regex
|
|
print("Cleaning text with regex...")
|
|
texts_df = pd.DataFrame({'text': [p['text'] for p in page_data_list]})
|
|
|
|
## Fix hyphenation
|
|
texts_df['text'] = texts_df['text'].str.replace(r'-\s*\n\s*', '', regex=True)
|
|
## Remove garbage OCR chars
|
|
texts_df['text'] = texts_df['text'].str.replace(r'[^a-zA-Z0-9\s.,;:\'"!?()-]', '', regex=True)
|
|
## Normalize spaces
|
|
texts_df['text'] = texts_df['text'].str.replace(r'\s+', ' ', regex=True)
|
|
|
|
## Reattach cleaned text
|
|
for i, clean_text in enumerate(texts_df['text'].tolist()):
|
|
page_data_list[i]['text'] = clean_text
|
|
|
|
## Save
|
|
print(f"Saving {len(page_data_list)} pages to JSON...")
|
|
with open(CLEANED_JSON, 'w', encoding='utf-8') as f:
|
|
json.dump(page_data_list, f, indent=2)
|
|
|
|
print(f"✅ Complete! {len(page_data_list)} clean pages saved.")
|
|
```
|
|
|
|
---
|
|
|
|
## CELL 5: BUILD VECTOR DB (GPU-Accelerated Embeddings)
|
|
|
|
```python
|
|
## @title BUILD VECTOR DB (30-60 min - may timeout, that's ok)
|
|
|
|
import json
|
|
import chromadb
|
|
from chromadb.utils import embedding_functions
|
|
|
|
print("Loading cleaned pages...")
|
|
with open(CLEANED_JSON, 'r', encoding='utf-8') as f:
|
|
pages_data = json.load(f)
|
|
|
|
print(f"Loaded {len(pages_data)} pages.")
|
|
|
|
## Initialize ChromaDB with GPU embeddings
|
|
print("Initializing ChromaDB with GPU embeddings...")
|
|
os.makedirs(VECTORDB_PATH, exist_ok=True)
|
|
|
|
gpu_ef = embedding_functions.SentenceTransformerEmbeddingFunction(
|
|
model_name="all-MiniLM-L6-v2",
|
|
device="cuda" # GPU accelerated
|
|
)
|
|
|
|
chroma_client = chromadb.PersistentClient(path=VECTORDB_PATH)
|
|
collection = chroma_client.get_or_create_collection(
|
|
name="historical_news",
|
|
embedding_function=gpu_ef
|
|
)
|
|
|
|
## Chunking function
|
|
def chunk_text(text, chunk_size=800, overlap=150):
|
|
chunks = []
|
|
start = 0
|
|
while start < len(text):
|
|
end = min(start + chunk_size, len(text))
|
|
chunks.append(text[start:end])
|
|
start += chunk_size - overlap
|
|
return chunks
|
|
|
|
## Extract & chunk
|
|
documents = []
|
|
metadatas = []
|
|
ids = []
|
|
|
|
print("Chunking and preparing documents...")
|
|
for page in pages_data:
|
|
issue_id = page.get('issue_id', 'unknown')
|
|
page_num = page.get('page_number', 0)
|
|
text = page.get('text', '')
|
|
|
|
if not text.strip():
|
|
continue
|
|
|
|
chunks = chunk_text(text)
|
|
|
|
for chunk_idx, chunk_text_data in enumerate(chunks):
|
|
documents.append(chunk_text_data)
|
|
metadatas.append({
|
|
"issue_id": issue_id,
|
|
"page_number": page_num,
|
|
"chunk_id": chunk_idx
|
|
})
|
|
ids.append(f"{issue_id}_p{page_num}_c{chunk_idx}")
|
|
|
|
total_chunks = len(documents)
|
|
print(f"Prepared {total_chunks} chunks.")
|
|
|
|
## Save checkpoint before starting (in case it times out)
|
|
checkpoint = {
|
|
'total_chunks': total_chunks,
|
|
'vectorized': 0,
|
|
'started_at': str(__import__('datetime').datetime.now())
|
|
}
|
|
|
|
checkpoint_file = os.path.join(VECTORDB_PATH, 'progress.json')
|
|
|
|
print("Adding to vector DB...")
|
|
BATCH_SIZE = 250
|
|
|
|
for b in range(0, total_chunks, BATCH_SIZE):
|
|
try:
|
|
collection.add(
|
|
documents=documents[b:b+BATCH_SIZE],
|
|
metadatas=metadatas[b:b+BATCH_SIZE],
|
|
ids=ids[b:b+BATCH_SIZE]
|
|
)
|
|
checkpoint['vectorized'] = b + BATCH_SIZE
|
|
|
|
if (b + BATCH_SIZE) % 1000 == 0 or (b + BATCH_SIZE) >= total_chunks:
|
|
with open(checkpoint_file, 'w') as f:
|
|
json.dump(checkpoint, f)
|
|
print(f"✓ {b + BATCH_SIZE}/{total_chunks} chunks vectorized")
|
|
|
|
except Exception as e:
|
|
print(f"❌ Error at batch {b}: {e}")
|
|
print(f"Saved progress checkpoint. DB is recoverable.")
|
|
break
|
|
|
|
print(f"✅ Vector DB complete! Stored at: {VECTORDB_PATH}")
|
|
print(f"Query it with: chroma_client.get_collection('historical_news')")
|
|
```
|
|
|
|
---
|
|
|
|
## CELL 6: TEST THE VECTOR DB (Optional)
|
|
|
|
```python
|
|
## @title TEST QUERY
|
|
|
|
import chromadb
|
|
|
|
db_path = VECTORDB_PATH
|
|
client = chromadb.PersistentClient(path=db_path)
|
|
collection = client.get_collection("historical_news")
|
|
|
|
## Test query
|
|
test_query = input("Enter a search term (e.g., 'railroad accident'): ").strip()
|
|
|
|
results = collection.query(
|
|
query_texts=[test_query],
|
|
n_results=5
|
|
)
|
|
|
|
print(f"\n🔍 Top 5 results for: '{test_query}'\n")
|
|
for i, (doc, metadata, distance) in enumerate(zip(
|
|
results['documents'][0],
|
|
results['metadatas'][0],
|
|
results['distances'][0]
|
|
)):
|
|
print(f"{i+1}. Issue: {metadata['issue_id']}, Page {metadata['page_number']}")
|
|
print(f" Relevance score: {1 - distance:.2f}")
|
|
print(f" Text: {doc[:150]}...\n")
|
|
```
|
|
|
|
---
|
|
|
|
## KEY CHANGES FROM ORIGINAL
|
|
|
|
| Issue | Original | Fixed |
|
|
|-------|----------|-------|
|
|
| **cudf** | ❌ Will crash | ✅ Use pandas (faster anyway) |
|
|
| **Drive mounts** | 3x (wasteful) | 1x (efficient) |
|
|
| **GPU usage** | Regex on GPU (slow) | Embeddings on GPU (fast) |
|
|
| **Hardcoded paths** | ❌ Not flexible | ✅ User input + parameterized |
|
|
| **Error handling** | Silent fails | ✅ Logged failures |
|
|
| **Checkpointing** | None | ✅ Progress saved |
|
|
| **Workers** | 4 (risky) | 2 (safe) |
|
|
| **Test mode** | N/A | ✅ Optional test with 50 issues |
|
|
| **Memory** | All in RAM | ✅ Batch processing |
|
|
|
|
---
|
|
|
|
## EXPECTED RESULTS
|
|
|
|
- **Download:** 30-45 min (1756 issues)
|
|
- **Parse & Clean:** 10-15 min
|
|
- **Vectorize:** 45-90 min (may timeout on Free Tier around 70 min mark)
|
|
- **Total:** ~2.5 hours
|
|
|
|
If vectorization times out, your progress is saved and resumable in next session.
|