vault backup: 2026-08-06 15:22:08
This commit is contained in:
shit-vault
@@ -0,0 +1,356 @@
|
||||
---
|
||||
created: 2026-08-06T15:15:00
|
||||
tags:
|
||||
- NotebookLM
|
||||
- colab
|
||||
- vector
|
||||
updated: 2026-08-06T15:15:00
|
||||
---
|
||||
|
||||
# REFACTORED COLAB NOTEBOOK - Free Tier Compatible
|
||||
## Kentucky Kernel Archive → Vector DB Pipeline
|
||||
|
||||
---
|
||||
|
||||
## CELL 1: SETUP & MOUNT (DO THIS FIRST)
|
||||
|
||||
```python
|
||||
## @title SETUP (RUN ONCE)
|
||||
|
||||
## Install everything upfront
|
||||
!pip install internetarchive chromadb sentence-transformers
|
||||
|
||||
## Mount Drive once, reuse everywhere
|
||||
from google.colab import drive
|
||||
drive.mount('/content/drive')
|
||||
|
||||
print("✅ Setup complete. Drive mounted.")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CELL 2: CONFIGURE PATHS (USER INPUT)
|
||||
|
||||
```python
|
||||
## @title CONFIG - Enter Your Settings
|
||||
|
||||
## User inputs
|
||||
creator_name = input("Enter creator name (e.g., 'The Mt. Sterling Advocate'): ").strip()
|
||||
test_mode = input("Test mode? (y/n, tests with 50 issues): ").strip().lower() == 'y'
|
||||
|
||||
## Paths
|
||||
BASE_DOWNLOAD = f'/content/{creator_name.replace(" ", "_")}_Archive'
|
||||
CLEANED_JSON = f'/content/drive/MyDrive/{creator_name.replace(" ", "_")}_Cleaned/cleaned_pages.json'
|
||||
VECTORDB_PATH = f'/content/drive/MyDrive/{creator_name.replace(" ", "_")}_VectorDB'
|
||||
|
||||
## Create directories
|
||||
import os
|
||||
os.makedirs(os.path.dirname(CLEANED_JSON), exist_ok=True)
|
||||
os.makedirs(BASE_DOWNLOAD, exist_ok=True)
|
||||
|
||||
print(f"✓ Creator: {creator_name}")
|
||||
print(f"✓ Download to: {BASE_DOWNLOAD}")
|
||||
print(f"✓ Cleaned JSON: {CLEANED_JSON}")
|
||||
print(f"✓ Vector DB: {VECTORDB_PATH}")
|
||||
print(f"✓ Test Mode: {test_mode}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CELL 3: DOWNLOAD FROM ARCHIVE.ORG
|
||||
|
||||
```python
|
||||
## @title DOWNLOAD (This takes 30-45 min)
|
||||
|
||||
from internetarchive import search_items, get_item, get_session
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
ia_session = get_session()
|
||||
ia_session.mount_http_adapter()
|
||||
|
||||
## Search
|
||||
query = f'creator:"{creator_name}"'
|
||||
print(f"Searching: {query}")
|
||||
|
||||
search = search_items(query, archive_session=ia_session)
|
||||
identifiers = [result['identifier'] for result in search]
|
||||
|
||||
if test_mode:
|
||||
identifiers = identifiers[:50] # Test with 50 only
|
||||
|
||||
total_items = len(identifiers)
|
||||
print(f"Found {total_items} issues. Starting download with 2 workers (safe)...\n")
|
||||
|
||||
def fast_download(identifier):
|
||||
target_folder = os.path.join(BASE_DOWNLOAD, identifier)
|
||||
if os.path.exists(target_folder) and len(os.listdir(target_folder)) >= 3:
|
||||
return f"⏭️ Skipped: {identifier}"
|
||||
|
||||
try:
|
||||
item = get_item(identifier, archive_session=ia_session)
|
||||
item.download(
|
||||
destdir=BASE_DOWNLOAD,
|
||||
glob_pattern=['*meta.xml', '*djvu.xml', '*djvu.txt'],
|
||||
ignore_existing=True,
|
||||
retries=3
|
||||
)
|
||||
return f"✅ Downloaded: {identifier}"
|
||||
except Exception as e:
|
||||
return f"❌ Failed: {identifier} - {str(e)[:50]}"
|
||||
|
||||
## Use 2 workers (safer than 4 on Free Tier)
|
||||
completed = 0
|
||||
failed = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
futures = {executor.submit(fast_download, identifier): identifier for identifier in identifiers}
|
||||
|
||||
for future in as_completed(futures):
|
||||
completed += 1
|
||||
result = future.result()
|
||||
if "❌" in result:
|
||||
failed.append(result)
|
||||
|
||||
if completed % 25 == 0 or completed == total_items:
|
||||
print(f"[{completed}/{total_items}] {result[:60]}")
|
||||
|
||||
print(f"\n✅ Download complete. {len(failed)} failures.")
|
||||
if failed:
|
||||
print("Failed items:")
|
||||
for f in failed[:5]:
|
||||
print(f" {f}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CELL 4: PARSE XML & CLEAN TEXT (CPU-BASED)
|
||||
|
||||
```python
|
||||
## @title PARSE & CLEAN
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
import json
|
||||
import pandas as pd # ← Use pandas, not cudf
|
||||
import glob
|
||||
|
||||
print("Scanning for DJVU XML files...")
|
||||
djvu_files = glob.glob(os.path.join(BASE_DOWNLOAD, '**/*djvu.xml'), recursive=True)
|
||||
print(f"Found {len(djvu_files)} XML files.")
|
||||
|
||||
page_data_list = []
|
||||
failed_issues = []
|
||||
|
||||
for djvu_path in djvu_files:
|
||||
folder_name = os.path.basename(os.path.dirname(djvu_path))
|
||||
|
||||
try:
|
||||
tree = ET.parse(djvu_path)
|
||||
root = tree.getroot()
|
||||
pages = root.findall('.//OBJECT')
|
||||
|
||||
for page_index, page in enumerate(pages):
|
||||
page_text = []
|
||||
for word in page.findall('.//WORD'):
|
||||
if word.text:
|
||||
page_text.append(word.text)
|
||||
|
||||
raw_page = " ".join(page_text)
|
||||
|
||||
if raw_page.strip():
|
||||
page_data_list.append({
|
||||
"issue_id": folder_name,
|
||||
"page_number": page_index + 1,
|
||||
"text": raw_page
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
failed_issues.append((folder_name, str(e)))
|
||||
|
||||
print(f"Extracted {len(page_data_list)} pages, {len(failed_issues)} failures.")
|
||||
|
||||
## CLEANING: Use pandas (CPU) - faster than GPU for regex
|
||||
print("Cleaning text with regex...")
|
||||
texts_df = pd.DataFrame({'text': [p['text'] for p in page_data_list]})
|
||||
|
||||
## Fix hyphenation
|
||||
texts_df['text'] = texts_df['text'].str.replace(r'-\s*\n\s*', '', regex=True)
|
||||
## Remove garbage OCR chars
|
||||
texts_df['text'] = texts_df['text'].str.replace(r'[^a-zA-Z0-9\s.,;:\'"!?()-]', '', regex=True)
|
||||
## Normalize spaces
|
||||
texts_df['text'] = texts_df['text'].str.replace(r'\s+', ' ', regex=True)
|
||||
|
||||
## Reattach cleaned text
|
||||
for i, clean_text in enumerate(texts_df['text'].tolist()):
|
||||
page_data_list[i]['text'] = clean_text
|
||||
|
||||
## Save
|
||||
print(f"Saving {len(page_data_list)} pages to JSON...")
|
||||
with open(CLEANED_JSON, 'w', encoding='utf-8') as f:
|
||||
json.dump(page_data_list, f, indent=2)
|
||||
|
||||
print(f"✅ Complete! {len(page_data_list)} clean pages saved.")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CELL 5: BUILD VECTOR DB (GPU-Accelerated Embeddings)
|
||||
|
||||
```python
|
||||
## @title BUILD VECTOR DB (30-60 min - may timeout, that's ok)
|
||||
|
||||
import json
|
||||
import chromadb
|
||||
from chromadb.utils import embedding_functions
|
||||
|
||||
print("Loading cleaned pages...")
|
||||
with open(CLEANED_JSON, 'r', encoding='utf-8') as f:
|
||||
pages_data = json.load(f)
|
||||
|
||||
print(f"Loaded {len(pages_data)} pages.")
|
||||
|
||||
## Initialize ChromaDB with GPU embeddings
|
||||
print("Initializing ChromaDB with GPU embeddings...")
|
||||
os.makedirs(VECTORDB_PATH, exist_ok=True)
|
||||
|
||||
gpu_ef = embedding_functions.SentenceTransformerEmbeddingFunction(
|
||||
model_name="all-MiniLM-L6-v2",
|
||||
device="cuda" # GPU accelerated
|
||||
)
|
||||
|
||||
chroma_client = chromadb.PersistentClient(path=VECTORDB_PATH)
|
||||
collection = chroma_client.get_or_create_collection(
|
||||
name="historical_news",
|
||||
embedding_function=gpu_ef
|
||||
)
|
||||
|
||||
## Chunking function
|
||||
def chunk_text(text, chunk_size=800, overlap=150):
|
||||
chunks = []
|
||||
start = 0
|
||||
while start < len(text):
|
||||
end = min(start + chunk_size, len(text))
|
||||
chunks.append(text[start:end])
|
||||
start += chunk_size - overlap
|
||||
return chunks
|
||||
|
||||
## Extract & chunk
|
||||
documents = []
|
||||
metadatas = []
|
||||
ids = []
|
||||
|
||||
print("Chunking and preparing documents...")
|
||||
for page in pages_data:
|
||||
issue_id = page.get('issue_id', 'unknown')
|
||||
page_num = page.get('page_number', 0)
|
||||
text = page.get('text', '')
|
||||
|
||||
if not text.strip():
|
||||
continue
|
||||
|
||||
chunks = chunk_text(text)
|
||||
|
||||
for chunk_idx, chunk_text_data in enumerate(chunks):
|
||||
documents.append(chunk_text_data)
|
||||
metadatas.append({
|
||||
"issue_id": issue_id,
|
||||
"page_number": page_num,
|
||||
"chunk_id": chunk_idx
|
||||
})
|
||||
ids.append(f"{issue_id}_p{page_num}_c{chunk_idx}")
|
||||
|
||||
total_chunks = len(documents)
|
||||
print(f"Prepared {total_chunks} chunks.")
|
||||
|
||||
## Save checkpoint before starting (in case it times out)
|
||||
checkpoint = {
|
||||
'total_chunks': total_chunks,
|
||||
'vectorized': 0,
|
||||
'started_at': str(__import__('datetime').datetime.now())
|
||||
}
|
||||
|
||||
checkpoint_file = os.path.join(VECTORDB_PATH, 'progress.json')
|
||||
|
||||
print("Adding to vector DB...")
|
||||
BATCH_SIZE = 250
|
||||
|
||||
for b in range(0, total_chunks, BATCH_SIZE):
|
||||
try:
|
||||
collection.add(
|
||||
documents=documents[b:b+BATCH_SIZE],
|
||||
metadatas=metadatas[b:b+BATCH_SIZE],
|
||||
ids=ids[b:b+BATCH_SIZE]
|
||||
)
|
||||
checkpoint['vectorized'] = b + BATCH_SIZE
|
||||
|
||||
if (b + BATCH_SIZE) % 1000 == 0 or (b + BATCH_SIZE) >= total_chunks:
|
||||
with open(checkpoint_file, 'w') as f:
|
||||
json.dump(checkpoint, f)
|
||||
print(f"✓ {b + BATCH_SIZE}/{total_chunks} chunks vectorized")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error at batch {b}: {e}")
|
||||
print(f"Saved progress checkpoint. DB is recoverable.")
|
||||
break
|
||||
|
||||
print(f"✅ Vector DB complete! Stored at: {VECTORDB_PATH}")
|
||||
print(f"Query it with: chroma_client.get_collection('historical_news')")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CELL 6: TEST THE VECTOR DB (Optional)
|
||||
|
||||
```python
|
||||
## @title TEST QUERY
|
||||
|
||||
import chromadb
|
||||
|
||||
db_path = VECTORDB_PATH
|
||||
client = chromadb.PersistentClient(path=db_path)
|
||||
collection = client.get_collection("historical_news")
|
||||
|
||||
## Test query
|
||||
test_query = input("Enter a search term (e.g., 'railroad accident'): ").strip()
|
||||
|
||||
results = collection.query(
|
||||
query_texts=[test_query],
|
||||
n_results=5
|
||||
)
|
||||
|
||||
print(f"\n🔍 Top 5 results for: '{test_query}'\n")
|
||||
for i, (doc, metadata, distance) in enumerate(zip(
|
||||
results['documents'][0],
|
||||
results['metadatas'][0],
|
||||
results['distances'][0]
|
||||
)):
|
||||
print(f"{i+1}. Issue: {metadata['issue_id']}, Page {metadata['page_number']}")
|
||||
print(f" Relevance score: {1 - distance:.2f}")
|
||||
print(f" Text: {doc[:150]}...\n")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## KEY CHANGES FROM ORIGINAL
|
||||
|
||||
| Issue | Original | Fixed |
|
||||
|-------|----------|-------|
|
||||
| **cudf** | ❌ Will crash | ✅ Use pandas (faster anyway) |
|
||||
| **Drive mounts** | 3x (wasteful) | 1x (efficient) |
|
||||
| **GPU usage** | Regex on GPU (slow) | Embeddings on GPU (fast) |
|
||||
| **Hardcoded paths** | ❌ Not flexible | ✅ User input + parameterized |
|
||||
| **Error handling** | Silent fails | ✅ Logged failures |
|
||||
| **Checkpointing** | None | ✅ Progress saved |
|
||||
| **Workers** | 4 (risky) | 2 (safe) |
|
||||
| **Test mode** | N/A | ✅ Optional test with 50 issues |
|
||||
| **Memory** | All in RAM | ✅ Batch processing |
|
||||
|
||||
---
|
||||
|
||||
## EXPECTED RESULTS
|
||||
|
||||
- **Download:** 30-45 min (1756 issues)
|
||||
- **Parse & Clean:** 10-15 min
|
||||
- **Vectorize:** 45-90 min (may timeout on Free Tier around 70 min mark)
|
||||
- **Total:** ~2.5 hours
|
||||
|
||||
If vectorization times out, your progress is saved and resumable in next session.
|
||||
Reference in New Issue
Block a user