stateofshit
7f189254c0
- 000-configs/notebooks/docx_to_markdown_openwebui.ipynb (29 cells) - mammoth + python-docx fallback, manifest dedup (SHA-256), pre-flight fuzzy dedup (Jaccard @ 0.97), URL extraction, gzip download - 000-configs/notebooks/README.md: index of notebooks + how-to-add - 000-configs/README.md: list notebooks/ subfolder in the table - 07-tasks/docx-to-markdown-converter-notebook.md: full task writeup
65 KiB
65 KiB
In [ ]:
# Run once per session. ~5–10 seconds on free tier.
%pip -q install mammoth==1.9.1 markdownify python-docx==1.1.2 requests beautifulsoup4In [ ]:
import os, shutil, csv, hashlib, re, datetime, time
from pathlib import Path
from google.colab import drive
drive.mount('/content/drive')
DRIVE_ROOT = Path('/content/drive/MyDrive/to_convert')
ARCHIVE_DIR = DRIVE_ROOT / 'archive'
MD_OUTPUT_DIR = DRIVE_ROOT / 'markdown_output'
DOWNLOADS_DIR = DRIVE_ROOT / 'downloads'
DOWNLOADS_CSV = DOWNLOADS_DIR / 'downloads.csv'
MANIFEST_PATH = DRIVE_ROOT / 'manifest.csv'
SOURCES_PATH = DRIVE_ROOT / 'sources.csv'
# Create the folder tree (mkdir -p). User just creates /to_convert/ once on Drive.
for p in (DRIVE_ROOT, ARCHIVE_DIR, MD_OUTPUT_DIR,
DOWNLOADS_DIR, DOWNLOADS_DIR / 'pdf',
DOWNLOADS_DIR / 'html', DOWNLOADS_DIR / 'other'):
p.mkdir(parents=True, exist_ok=True)
# Local scratch input dir (wiped each run so uploads don't accumulate).
INPUT_DIR = Path('/content/docx_input')
if INPUT_DIR.exists():
shutil.rmtree(INPUT_DIR)
INPUT_DIR.mkdir(parents=True, exist_ok=True)
print(f'Drive root: {DRIVE_ROOT}')
print(f'Local input: {INPUT_DIR}')
print(f'MD output: {MD_OUTPUT_DIR}')In [ ]:
import os, shutil, tarfile
from pathlib import Path
from google.colab import files as colab_files
# Edit this to point at a Drive folder that holds your archive, or comment it out to use the upload fallback.
DRIVE_ARCHIVE_SOURCE_DIR = Path('/content/drive/MyDrive/zzz-new_shit')
tar_archive_path = None
if DRIVE_ARCHIVE_SOURCE_DIR.exists():
for item in DRIVE_ARCHIVE_SOURCE_DIR.glob('*.tar'):
if item.is_file():
tar_archive_path = item
break
if not tar_archive_path:
for item in DRIVE_ARCHIVE_SOURCE_DIR.glob('*.tgz'):
if item.is_file():
tar_archive_path = item
break
if tar_archive_path:
print(f"Found archive: {tar_archive_path.name}")
temp_archive_path = Path('/content') / tar_archive_path.name
print(f"Copying '{tar_archive_path}' to '{temp_archive_path}'...")
shutil.copy2(tar_archive_path, temp_archive_path)
print("Copy complete.")
print(f"Extracting '{temp_archive_path}' to '{INPUT_DIR}'...")
with tarfile.open(temp_archive_path, "r") as tar:
tar.extractall(path=INPUT_DIR, filter="data")
print("Extraction complete.")
temp_archive_path.unlink()
else:
print(f"No archive in {DRIVE_ARCHIVE_SOURCE_DIR} — using Colab upload picker.")
print("Pick .docx and/or .md files in the dialog that appears.")
uploaded = colab_files.upload()
if not uploaded:
raise RuntimeError("No files uploaded.")
for name, data in uploaded.items():
(INPUT_DIR / name).write_bytes(data)
print(f"Uploaded {len(uploaded)} file(s) into {INPUT_DIR}.")
# Discover inputs (recursively, in case the tar extracted into subdirs).
input_files = sorted([*INPUT_DIR.glob('**/*.docx'), *INPUT_DIR.glob('**/*.md')])
n_docx = sum(1 for f in input_files if f.suffix.lower() == '.docx')
n_md = sum(1 for f in input_files if f.suffix.lower() == '.md')
print(f"\nFound {len(input_files)} input file(s): {n_docx} .docx, {n_md} .md")
for f in input_files:
print(' •', f.relative_to(INPUT_DIR))
# Snapshot the original upload list so the pre-flight dedup cell (5b) can be
# re-run safely without the list shrinking on each pass (idempotent).
uploaded_files_master = list(input_files)In [ ]:
# === Config ===
SKIP_ALREADY_SEEN = True # if True, skip files whose sha256 is already in manifest.csv
ARCHIVE_INPUTS = True # copy each input's bytes to archive/ for future hash matching
DOWNLOAD_SOURCES = False # only the explicit downloader cell flips this on
RATE_LIMIT_SEC = 1.0 # politeness delay between URL fetches
# Pre-flight near-duplicate detection (cell 5b). Two files count as 'the same doc'
# when their overlapping-fragment ratio is >= this threshold. 0.97 means '~95-97%'
# chunk overlap leaves room for tiny edits (a single cite marker, a timestamp).
# Set to 1.0 to disable fuzzy dedup entirely (then only exact byte-hashes dedup).
# Lower to 0.90 to be more aggressive (riskier -- might merge docs that share a lot
# of boilerplate but ARE different).
DEDUP_SIMILARITY_THRESHOLD = 0.97
DEDUP_CHUNK_SIZE = 500 # chars per fragment in the overlapping-fragment signature
# Optional Mammoth style map. Maps Word paragraph styles to HTML headings
# so mammoth emits real <h1>/<h2>/<h3> instead of generic <p>. The converter's
# collapse_extra_h1() then keeps the first H1 (the doc title) and demotes any
# additional H1s to H2 so every file has exactly ONE H1 (required by Open WebUI's
# Markdown Header Splitter, which treats each H1 as a new document section).
CUSTOM_STYLE_MAP = """
p[style-name='Title'] => h1:fresh
p[style-name='Subtitle'] => h2:fresh
p[style-name='Heading 1'] => h1:fresh
p[style-name='Heading 2'] => h2:fresh
p[style-name='Heading 3'] => h3:fresh
p[style-name='Heading 4'] => h4:fresh
"""
# If True, images are extracted to a per-doc `media/` folder and referenced
# in Markdown as !media/<name>. Open WebUI currently indexes text only, so
# inlined image tags are harmless but won't add embedding signal.
EXTRACT_IMAGES = TrueIn [ ]:
import csv, hashlib, re, datetime
from pathlib import Path
MANIFEST_COLS = ['sha256', 'first_seen_filename', 'first_seen_at',
'last_processed_at', 'input_ext', 'output_path',
'output_bytes', 'status']
SOURCES_COLS = ['doc_sha256', 'doc_stem', 'url', 'url_type']
DOWNLOADS_COLS = ['url', 'sha256_of_bytes', 'saved_path',
'http_status', 'content_type', 'downloaded_at']
def sha256_of_file(path: Path) -> str:
"""Stream-hash a file in 1MB chunks; returns hex digest."""
h = hashlib.sha256()
with open(path, 'rb') as f:
for chunk in iter(lambda: f.read(1024 * 1024), b''):
h.update(chunk)
return h.hexdigest()
def _now_iso() -> str:
return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')
def load_manifest() -> list:
if not MANIFEST_PATH.exists():
return []
with open(MANIFEST_PATH, 'r', newline='') as f:
return list(csv.DictReader(f))
def append_manifest_row(row: dict):
exists = MANIFEST_PATH.exists()
with open(MANIFEST_PATH, 'a', newline='') as f:
w = csv.DictWriter(f, fieldnames=MANIFEST_COLS)
if not exists:
w.writeheader()
w.writerow({k: row.get(k, '') for k in MANIFEST_COLS})
def is_seen(sha256: str, manifest: list):
for r in manifest:
if r.get('sha256') == sha256:
return r
return None
_URL_RE = re.compile(r'https?://[^\s\)\]\>\}\,]+')
def extract_urls(text: str) -> list:
"""Extract full http(s):// URLs, stripping trailing punctuation."""
urls, seen = [], set()
for m in _URL_RE.findall(text):
u = m.rstrip('.,);:>]}>')
if u and u not in seen:
seen.add(u)
urls.append(u)
return urls
def classify_url(url: str) -> str:
"""Classify a URL by its path extension (no network call)."""
path = url.split('?')[0].split('#')[0].lower()
if path.endswith('.pdf'):
return 'pdf'
return 'html'
def append_source_rows(doc_sha256: str, doc_stem: str, urls: list):
exists = SOURCES_PATH.exists()
with open(SOURCES_PATH, 'a', newline='') as f:
w = csv.DictWriter(f, fieldnames=SOURCES_COLS)
if not exists:
w.writeheader()
for u in urls:
w.writerow({'doc_sha256': doc_sha256, 'doc_stem': doc_stem,
'url': u, 'url_type': classify_url(u)})
# --- Pre-flight fuzzy-dedup helpers -------------------------------------------
# Two files count as "the same document" when their overlapping-fragment ratio
# is >= DEDUP_SIMILARITY_THRESHOLD. We extract raw text fast (python-docx for
# .docx, plain read for .md), normalize away everything that's not 'real text'
# (whitespace, punctuation, urls, the gemini masthead, leading '# title'), and
# split into DEDUP_CHUNK_SIZE-character sliding windows keyed by sha-256. The
# Jaccard similarity of those two sets ~= fraction of overlapping content.
DEDUP_DROPS_CSV = DRIVE_ROOT / 'dedup_drops.csv'
DEDUP_DROPS_COLS = ['kept_filename', 'kept_sha256', 'dropped_filename',
'dropped_sha256', 'similarity', 'drop_reason', 'dropped_at']
# Lightweight fallback for extracting text from a .docx WITHOUT running mammoth
# (which is slow and can crash). Uses python-docx (already installed). Returns
# an empty string on any failure so the file is still hashed and compared but
# with effectively no content signal.
try:
import docx as _pdocx_dedupe
except Exception:
_pdocx_dedupe = None
def _raw_text_for_dedup(path: Path) -> str:
"""Get a raw-text signal from a .docx or .md for near-duplicate comparison.
On any failure returns '' - the caller will then fall back to byte-hash dedup.
"""
ext = path.suffix.lower()
try:
if ext == '.md':
return path.read_text(encoding='utf-8', errors='replace')
if ext == '.docx':
if _pdocx_dedupe is None:
return ''
d = _pdocx_dedupe.Document(str(path))
parts = []
for p in d.paragraphs:
t = p.text
if t:
parts.append(t)
# also pull table text (some real content lives there)
for tbl in d.tables:
for row in tbl.rows:
for cell in row.cells:
for p in cell.paragraphs:
if p.text:
parts.append(p.text)
return '\n'.join(parts)
except Exception:
return ''
return ''
_NORM_WS = re.compile(r'\s+')
_NORM_PUNCT = re.compile(r'[^a-z0-9 ]')
_URL_STRIP_RE = re.compile(r'https?://\S+')
_MASTHEAD_STRIP_RE = re.compile(r'Created By:.*?DATE:[^\n]*', re.I | re.S)
def _normalize_for_dedup(text: str) -> str:
"""Aggressively normalize for robust near-duplicate comparison.
Removes: URLs (they make every research doc look 100% like another),
the Gemini masthead block, whitespace, and punctuation. Lowercases.
"""
if not text:
return ''
text = _URL_STRIP_RE.sub(' ', text)
text = _MASTHEAD_STRIP_RE.sub(' ', text)
text = text.lower()
text = _NORM_PUNCT.sub(' ', text)
text = _NORM_WS.sub(' ', text).strip()
return text
def _chunk_set(text: str, chunk_size: int) -> set:
"""Build a set of sha-256 hex digests, one per non-empty non-overlapping
chunk_size-char window. Used as a fast Jaccard signature.
"""
if not text:
return set()
chunks = set()
i = 0
n = len(text)
while i < n:
piece = text[i:i + chunk_size].strip()
if piece:
chunks.add(hashlib.sha256(piece.encode('utf-8')).hexdigest())
i += chunk_size
return chunks
def _jaccard(a: set, b: set) -> float:
"""Jaccard overlap ratio. Returns 0.0 for two empty sets (treat as not-same)."""
if not a or not b:
return 0.0
inter = len(a & b)
union = len(a | b)
return inter / union if union else 0.0
def append_dedup_drop_row(row: dict):
exists = DEDUP_DROPS_CSV.exists()
with open(DEDUP_DROPS_CSV, 'a', newline='') as f:
w = csv.DictWriter(f, fieldnames=DEDUP_DROPS_COLS)
if not exists:
w.writeheader()
w.writerow({k: row.get(k, '') for k in DEDUP_DROPS_COLS})
In [ ]:
import mammoth
from markdownify import markdownify as md
import re
from pathlib import Path
def strip_masthead(text: str) -> str:
"""Remove the Gemini Notebook authorship block.
Every Gemini-authored DOCX starts with a masthead like:
—
Created By: <author>
APP: Gemini Notebook
DATE: <date>
—
(sometimes bold-wrapped). It is pure authorship metadata with zero retrieval
value; we drop it along with its surrounding standalone em-dash dividers.
Pattern-based so it works regardless of where it sits in the pipeline.
"""
lines = text.split('\n')
MASTHEAD_RE = re.compile(r'^(\*\*)?(Created By:|APP:\s*Gemini|DATE:)', re.I)
EMDASH_RE = re.compile(r'^(\*\*)?—(\*\*)?$')
drop_idx = set()
for i, ln in enumerate(lines):
s = ln.strip()
if MASTHEAD_RE.match(s):
# drop this masthead line, plus adjacent em-dash dividers
drop_idx.add(i)
# scan outward for wrapping em-dashes / blank-bounded dashes
j = i - 1
while j >= 0 and (lines[j].strip() == '' or EMDASH_RE.match(lines[j].strip())):
if EMDASH_RE.match(lines[j].strip()):
drop_idx.add(j)
break
j -= 1
j = i + 1
while j < len(lines) and (lines[j].strip() == '' or EMDASH_RE.match(lines[j].strip())):
if EMDASH_RE.match(lines[j].strip()):
drop_idx.add(j)
break
j += 1
# also fold in multi-line masthead (e.g. 'Created By: X\nAPP: Y' wrapped in **)
# by scanning forward through consecutive masthead/em-dash/blank lines
k = i + 1
while k < len(lines):
ks = lines[k].strip()
if MASTHEAD_RE.match(ks) or EMDASH_RE.match(ks) or ks == '':
if MASTHEAD_RE.match(ks) or EMDASH_RE.match(ks):
drop_idx.add(k)
k += 1
else:
break
if not drop_idx:
return text
out = [ln for idx, ln in enumerate(lines) if idx not in drop_idx]
# collapse any triple+ blanks created by removal
return re.sub(r'\n{3,}', '\n\n', '\n'.join(out))
def _make_converter(extract_images: bool, out_dir: Path, doc_stem: str):
"""Build a mammoth image converter that saves media to disk."""
if not extract_images:
return None
media_dir = out_dir / doc_stem / 'media'
media_dir.mkdir(parents=True, exist_ok=True)
counter = {'n': 0}
def convert_image(image):
counter['n'] += 1
ext = (image.content_type or 'image/png').split('/')[-1].replace('jpeg', 'jpg')
fname = f"image_{counter['n']:03d}.{ext}"
with image.open() as f:
(media_dir / fname).write_bytes(f.read())
rel = f"{doc_stem}/media/{fname}"
return {'src': rel}
return convert_image
def clean_markdown(text: str) -> str:
"""Remove Word conversion artifacts and normalize whitespace."""
text = re.sub(r'<a[^>]*></a>', '', text)
text = re.sub(r'<a\s+(?:[^>]*?)>', '', text)
text = text.replace('</a>', '')
text = re.sub(r'\n{3,}', '\n\n', text)
text = '\n'.join(line.rstrip() for line in text.splitlines())
text = re.sub(r'(?m)^#{1,6}\s*$', '', text)
return text.strip() + '\n'
def strip_cite_markers(text: str) -> str:
"""Remove Gemini Notebook citation artifacts like '[cite: 1, 2, 3]'.
These reference internal citation numbers that mean nothing to an LLM
or embedding model and just waste tokens / add noise to retrieval."""
return re.sub(r'\s*\[cite:\s*[\d,\s]+\]', '', text)
def collapse_hr(text: str) -> str:
"""Replace 20+ dash horizontal-rule lines with a sentinel '<HR>' we can
key section boundaries off of, and drop them as content (a bare HR would
just become dead weight inside Open WebUI chunks)."""
return re.sub(r'(?m)^-{20,}$', '<HR>', text)
def promote_structure(text: str) -> str:
"""Promote implicit Word structure into real Markdown headings.
These Gemini-authored DOCX files use bold-only paragraphs (NOT Word
'Heading 1/2' styles) for section titles, and a row of dashes as a
top-level section divider. Mammoth therefore emits them as <strong>/bold,
and Open WebUI's Markdown Header Splitter would have nothing to split on.
Rules:
- A standalone line fully wrapped in **...** (and NOT inside a table,
i.e. not starting with '|'), <=160 chars and not a bare '—'
-> candidate heading.
- A bold heading immediately after a former HR -> '## ' (H2 section).
- Any other bold heading -> '### ' (H3 subsection).
- Table-cell bold stays inside '| ... |' rows and is never promoted.
- collapse_extra_h1() (later in the pipeline) is what guarantees
exactly one H1 per file, by keeping the first real H1 (or by
convert_one prepending a synthetic '# <stem>' if none exists).
"""
lines = text.split('\n')
out = []
after_hr = False
BOLD_RE = re.compile(r'^\*\*(.+?)\*\*$')
for ln in lines:
stripped = ln.strip()
if stripped == '<HR>':
after_hr = True
continue
if not stripped:
out.append(ln)
continue
m = BOLD_RE.match(stripped)
if m and not stripped.startswith('|'):
inner = m.group(1).strip()
# collapse adjacent '**A** **B** **C**' style runs into one title
inner = re.sub(r'\*\*\s*\*\*', ' ', inner)
inner = re.sub(r'\s+', ' ', inner).strip()
if len(inner) <= 160 and inner != '—':
# Promote this bold paragraph to a heading. The first bold line is
# NOT special-cased anymore: collapse_extra_h1() ensures exactly one
# H1 per file (the doc's real Title style or the synthetic '# <stem>'),
# so dropping the first bold would just lose a legitimate section
# header like 'Alpha Section'.
level = '##' if after_hr else '###'
out.append(f'{level} {inner}')
after_hr = False
continue
after_hr = False
out.append(ln)
return '\n'.join(out)
def collapse_extra_h1(text: str):
"""Ensure exactly one H1 per file.
Some source DOCX files use Word's 'Title' or 'Heading 1' paragraph style, which
mammoth converts to real <h1> -> '# ' headings. Combined with the synthetic
'# <stem>' H1 that convert_one prepends, this produces files with 2+ H1s.
Open WebUI's Markdown Header Splitter treats every H1 as the start of a new
document section, so multiple H1s fragment retrieval badly.
Returns (new_text, has_h1): if the doc already has at least one real H1, we keep
the first one and demote any subsequent H1s to H2; the caller then SKIPS prepending
the synthetic '# <stem>' title (since the doc already has a title H1).
"""
lines = text.split('\n')
h1_seen = False
out = []
for ln in lines:
m = re.match(r'^(#{1})\s+(.*)', ln)
if m:
if not h1_seen:
# Keep the first H1 as the document title.
h1_seen = True
out.append(ln)
else:
# Demote every subsequent H1 to H2 so there is exactly one H1.
out.append(f'## {m.group(2)}')
else:
out.append(ln)
return '\n'.join(out), h1_seen
def convert_one(docx_path: Path, out_root: Path, style_map: str, extract_images: bool):
"""Convert a single .docx to .md. Returns (out_file, warnings)."""
stem = docx_path.stem
with open(docx_path, 'rb') as f:
kwargs = dict()
if style_map:
kwargs['style_map'] = style_map
if extract_images:
kwargs['convert_image'] = _make_converter(True, out_root, stem)
result = mammoth.convert_to_html(f, **kwargs)
html = result.value
markdown = md(html, heading_style='ATX', bullets='-', strip=['a'])
# Pipeline order matters: strip noise -> collapse HR -> promote structure
markdown = clean_markdown(markdown)
markdown = strip_masthead(markdown)
markdown = strip_cite_markers(markdown)
markdown = collapse_hr(markdown)
markdown = promote_structure(markdown)
markdown, has_h1 = collapse_extra_h1(markdown)
# Synthetic H1 from the source filename stem gives every knowledge-base
# chunk a stable, cite-friendly document title (Open WebUI shows this).
if not has_h1:
# No real H1 in the source -> use the filename stem as the title.
markdown = f"# {stem}\n\n" + markdown.strip() + "\n"
else:
# Source already has an H1 (e.g. Word Title style); keep it as the doc title
# and do NOT prepend a duplicate synthetic H1 (would re-introduce the 2x-H1 bug).
markdown = markdown.strip() + "\n"
out_file = out_root / f"{stem}.md"
out_file.write_text(markdown, encoding='utf-8')
return out_file, result.messages
# --- Helper: python-docx fallback for corrupt docx that crashes mammoth ----------
try:
import docx as _pdocx # python-docx
except Exception:
_pdocx = None
def _convert_via_python_docx(docx_path: Path, out_file: Path, stem: str):
"""Fallback converter for docx files mammoth crashed on (e.g. the _accept0 bug).\n \n Extracts paragraph text + structure via python-docx. Differs from mammoth path:\n - Recognizes Word Heading 1/2/3/4 + Title styles as real markdown headings.\n - Detects code/CLI paragraphs (shell prompts, Python comments, etc.) and fences\n them in ``` blocks so leading "# comment" lines do NOT become faux H1 markdown\n headings (a bug that previously produced files with H1:6 / H1:15 / H1:279).\n - Runs collapse_extra_h1() as a safety net so the file ends up with exactly\n one H1, matching the contract of the primary convert_one() path.\n """
if _pdocx is None:
raise RuntimeError('python-docx not installed; cannot fallback')
d = _pdocx.Document(str(docx_path))
lines = []
h1_seen = False
in_code = False # whether we are currently inside a fenced code block
# Heuristics for detecting a paragraph that is really a code/CLI line, not prose.
# "Looks like code" -> preserve verbatim inside a fenced block so neither the
# leading '# ' (Python/shell comment) nor leading shell symbols become faux
# markdown headings.
import re as _re
def _looks_like_code(s: str) -> bool:
if not s:
return False
# Common shell prompt prefixes
if _re.match(r'^(\$|>#|>|\$\s|sudo |apt |pip |git |cd |npm |python|wget |curl |chmod |mkdir |cp |mv |rm |echo |export |systemctl |dpkg |sed |awk |grep |find |tar |unzip |./|npm |npx|docker |kubectl |helm |kubectl )', s):
return True
# Python-style comment (# ...) or shell comment. Excludes legitimate prose
# like '# Topic Idea:' only if it ALSO has code-y characters. To be safe,
# treat a line starting with '# ' followed by uppercase letters OR containing
# common code tokens as a code comment. We DON'T classify every '#'-leading
# line as code (would defeat real H1 detection); we only fence when it is
# clearly not a title. Real titles are short (<80 chars), Capitalized prose.
m = _re.match(r'^# (.{1,160})$', s)
if m and _re.search(r'[;|=&><{}\\/]|from \w|import \w|def \w|class \w|^if |^else|^for |^while |^return |=>|->|name ==', s):
return True
return False
for p in d.paragraphs:
txt = p.text.rstrip()
if not txt.strip():
if in_code:
# blank line inside a code block: keep it (preserves formatting)
lines.append('')
else:
lines.append('')
continue
style = (p.style.name or '').lower() if p.style else ''
# 1. Word heading styles -> real markdown headings
if 'heading 1' in style or style == 'title':
if in_code: lines.append('```'); in_code = False
if not h1_seen: lines.append(f'# {txt.strip()}'); h1_seen = True
else: lines.append(f'## {txt.strip()}')
elif 'heading 2' in style:
if in_code: lines.append('```'); in_code = False
lines.append(f'## {txt.strip()}')
elif 'heading 3' in style:
if in_code: lines.append('```'); in_code = False
lines.append(f'### {txt.strip()}')
elif 'heading 4' in style:
if in_code: lines.append('```'); in_code = False
lines.append(f'#### {txt.strip()}')
else:
# NOT a styled heading. Two sub-cases:
# (a) Bold-only short paragraph in a Normal style -> candidate H3 section header
is_bold = bool(p.runs) and all(r.bold for r in p.runs if r.text.strip())
if is_bold and len(txt.strip()) <= 160 and not txt.lstrip().startswith('#'):
if in_code: lines.append('```'); in_code = False
lines.append(f'### {txt.strip()}')
elif _looks_like_code(txt.strip()):
# (b) code/comment line -> accumulate inside a fenced block.
# Crucial: a '# foo' line inside ``` ``` is NOT a markdown H1.
if not in_code:
lines.append('```')
in_code = True
lines.append(txt)
else:
# plain prose paragraph -> close any open code block first
if in_code: lines.append('```'); in_code = False
lines.append(txt.strip())
if in_code: lines.append('```') # close trailing code block
body = '\n'.join(lines).strip()
if not h1_seen:
body = f'# {stem}\n\n' + body
# Safety net: ensure exactly one H1 (e.g. if doc has multiple 'Title' paragraphs,
# or a '# comment' paragraph slipped past the heuristics above).
body, _ = collapse_extra_h1(body)
out_file.write_text(body + '\n', encoding='utf-8')
return out_file, []
def clean_md_input(path: Path, out_file: Path, stem: str):
"""Cleanup pipeline for raw uploaded .md files.
Same post-processing as docx, minus mammoth (the file is already markdown):
clean_markdown -> strip_masthead -> strip_cite_markers
-> collapse_hr -> promote_structure -> collapse_extra_h1
-> conditional synthetic '# {stem}' H1 prepend.
"""
text = path.read_text(encoding='utf-8')
text = clean_markdown(text)
text = strip_masthead(text)
text = strip_cite_markers(text)
text = collapse_hr(text)
text = promote_structure(text)
text, has_h1 = collapse_extra_h1(text)
if not has_h1:
text = f"# {stem}\n\n" + text.strip() + "\n"
else:
text = text.strip() + "\n"
out_file.write_text(text, encoding='utf-8')
return out_file
In [ ]:
# Start from the snapshot so reruns are idempotent.
input_files = list(uploaded_files_master) if 'uploaded_files_master' in globals() \
else list(input_files)
if not DEDUP_SIMILARITY_THRESHOLD or DEDUP_SIMILARITY_THRESHOLD <= 0:
print(f"Near-duplicate scan: DISABLED (DEDUP_SIMILARITY_THRESHOLD={DEDUP_SIMILARITY_THRESHOLD})")
print(f"Proceeding with {len(input_files)} input file(s).")
else:
print(f"Near-duplicate scan (threshold: {DEDUP_SIMILARITY_THRESHOLD:.2f} chunk overlap)...")
print(f"Hashing {len(input_files)} input(s) into normalized text signatures...")
# Build a normalized signature for each input: raw byte-hash AND a normalized
# chunk set (used for the fuzzy comparison).
signatures = []
for path in input_files:
sha = sha256_of_file(path)
raw = _raw_text_for_dedup(path)
norm = _normalize_for_dedup(raw)
chunks = _chunk_set(norm, DEDUP_CHUNK_SIZE)
signatures.append({'path': path, 'sha': sha, 'chunks': chunks,
'n_chars': len(norm), 'n_chunks': len(chunks)})
# First pass: group by exact byte hash (instant, free, catches identical uploads).
by_sha = {}
for s in signatures:
by_sha.setdefault(s['sha'], []).append(s)
kept = [] # list of signature dicts we'll keep
dropped_exact = [] # (kept_sig, dup_sig)
for sha, group in by_sha.items():
if len(group) == 1:
kept.append(group[0])
else:
# Keep the first, drop the rest with similarity 1.00 (exact).
keep = group[0]
kept.append(keep)
for dup in group[1:]:
dropped_exact.append((keep, dup))
# Second pass: fuzzy compare across all 'kept' so far. For each candidate,
# compare its chunk set against every already-accepted doc whose chunk count
# is within a sane ratio (10x either way; wildly different doc sizes can't be
# 97% overlapping unless one is mostly empty, which we skip).
final_kept = []
dropped_fuzzy = [] # (kept_sig, dup_sig, similarity)
for cand in kept:
if not cand['chunks']:
# Couldn't extract text (failed docx parse, empty file). Keep it
# unconditionally; byte-hash dedup already handled exact dupes.
final_kept.append(cand)
continue
match = None
for keep in final_kept:
if not keep['chunks']:
continue
nc, nk = cand['n_chunks'], keep['n_chunks']
if nc == 0 or nk == 0:
continue
# Cheap pre-filter: can't be 97% similar if chunk counts are >10x apart.
if nc > 10 * nk or nk > 10 * nc:
continue
sim = _jaccard(cand['chunks'], keep['chunks'])
if sim >= DEDUP_SIMILARITY_THRESHOLD:
match = (keep, sim)
break
if match:
dropped_fuzzy.append((match[0], cand, match[1]))
else:
final_kept.append(cand)
# Build the new input_files list, preserving the original iteration order.
kept_paths = {s['path'] for s in final_kept}
input_files = [p for p in input_files if p in kept_paths]
# Report
print()
print(f"Exact-hash duplicates dropped: {len(dropped_exact)}")
for keep, dup in dropped_exact:
print(f" Keep '{keep['path'].name}'")
print(f" ⤷ dup '{dup['path'].name}' — identical bytes — dropped")
append_dedup_drop_row({
'kept_filename': keep['path'].name, 'kept_sha256': keep['sha'],
'dropped_filename': dup['path'].name, 'dropped_sha256': dup['sha'],
'similarity': '1.000', 'drop_reason': 'exact_byte_match',
'dropped_at': _now_iso()})
print()
print(f"Near-duplicate (fuzzy) drops: {len(dropped_fuzzy)}")
for keep, dup, sim in dropped_fuzzy:
print(f" Keep '{keep['path'].name}' ({keep['n_chunks']} chunks)")
print(f" ⤷ dup '{dup['path'].name}' — {sim*100:.1f}% overlap — dropped")
append_dedup_drop_row({
'kept_filename': keep['path'].name, 'kept_sha256': keep['sha'],
'dropped_filename': dup['path'].name, 'dropped_sha256': dup['sha'],
'similarity': f"{sim:.3f}", 'drop_reason': 'fuzzy_overlap',
'dropped_at': _now_iso()})
print()
print(f"Deduped: {len(uploaded_files_master)} → {len(input_files)} inputs "
f"({len(uploaded_files_master) - len(input_files)} duplicates dropped).")
print(f"Pre-flight complete. Cell 6 will now convert {len(input_files)} file(s).")
In [ ]:
manifest = load_manifest()
manifest_seen = {r['sha256'] for r in manifest}
converted_this_run = [] # list of (out_path, sha256, stem)
skipped_count = 0
crashed_count = 0
for path in input_files:
sha = sha256_of_file(path)
ext = path.suffix.lower().lstrip('.')
stem = path.stem
# Dedup check against the persistent Drive manifest. Guard against the rare
# race where sha is in manifest_seen (the set) but is_seen() returns None
# (e.g. another process appended to manifest.csv after we cached the set).
if SKIP_ALREADY_SEEN and sha in manifest_seen:
seen_row = is_seen(sha, manifest) or {}
prev_name = seen_row.get('first_seen_filename', '(unknown)')
prev_date = seen_row.get('first_seen_at', '(unknown)')
print(f"SKIP {path.name} (already converted; first seen as "
f"'{prev_name}' on {prev_date})")
skipped_count += 1
continue
# Archive original input bytes (durable across Colab wipes)
if ARCHIVE_INPUTS:
archive_path = ARCHIVE_DIR / f"{sha}.{ext}"
if not archive_path.exists():
shutil.copy2(path, archive_path)
out_file = MD_OUTPUT_DIR / f"{stem}.md"
status = 'converted'
try:
if ext == 'docx':
try:
out_file, warnings = convert_one(path, MD_OUTPUT_DIR,
CUSTOM_STYLE_MAP, EXTRACT_IMAGES)
except Exception:
out_file, warnings = _convert_via_python_docx(path, out_file, stem)
status = 'crashed_fallback_ok'
elif ext == 'md':
out_file = clean_md_input(path, out_file, stem)
status = 'converted'
else:
print(f"SKIP {path.name} (unsupported extension .{ext})")
continue
size = out_file.stat().st_size
rel_out = str(out_file.relative_to(DRIVE_ROOT))
row = {
'sha256': sha,
'first_seen_filename': path.name,
'first_seen_at': _now_iso(),
'last_processed_at': _now_iso(),
'input_ext': ext,
'output_path': rel_out,
'output_bytes': size,
'status': status,
}
append_manifest_row(row)
manifest_seen.add(sha)
converted_this_run.append((out_file, sha, stem))
print(f"✓ {path.name} → {out_file.name} ({size:,} bytes) [{status}]")
except Exception as e:
crashed_count += 1
row = {
'sha256': sha,
'first_seen_filename': path.name,
'first_seen_at': _now_iso(),
'last_processed_at': _now_iso(),
'input_ext': ext,
'output_path': '',
'output_bytes': 0,
'status': 'crashed_skipped',
}
append_manifest_row(row)
print(f"✗ {path.name} FAILED ({type(e).__name__}: {e})")
print(f"\nDone. {len(converted_this_run)} converted, "
f"{skipped_count} skipped (already in manifest), "
f"{crashed_count} crashed.")
print(f"Manifest now has {len(load_manifest())} total rows.")
In [ ]:
total_urls = 0
docs_with_urls = 0
for out_path, sha, stem in converted_this_run:
text = out_path.read_text(encoding='utf-8')
urls = extract_urls(text)
if urls:
append_source_rows(sha, stem, urls)
total_urls += len(urls)
docs_with_urls += 1
types = [classify_url(u) for u in urls]
n_pdf = types.count('pdf')
n_html = types.count('html')
n_other = types.count('other')
print(f" {stem[:70]}: {len(urls)} URLs ({n_pdf} pdf, {n_html} html, {n_other} other)")
else:
print(f" {stem[:70]}: 0 URLs")
print(f"\nTotal: {total_urls} URLs across {docs_with_urls} docs "
f"({len(converted_this_run) - docs_with_urls} had no URLs).")In [ ]:
from collections import Counter
import re
print("Heading structure per converted file (this run only):\n")
for out_path, sha, stem in converted_this_run:
counts = Counter()
text = out_path.read_text(encoding='utf-8')
for ln in text.splitlines():
m = re.match(r'^(#{1,6})\s', ln)
if m:
counts[len(m.group(1))] += 1
summary = ", ".join(f"H{lvl}:{counts[lvl]}" for lvl in sorted(counts))
flag_no = " ⚠ NO sub-headings!" if counts.get(2, 0) + counts.get(3, 0) == 0 else ""
flag_size = " ⚠ nearly empty" if len(text) < 500 else ""
flag_h1 = " ⚠ multiple H1s!" if counts.get(1, 0) > 1 else ""
print(f" {stem[:70]}")
print(f" {summary} ({len(text):,} chars){flag_no}{flag_size}{flag_h1}")
print(f"\nManifest total: {len(load_manifest())} files tracked since first run.")In [ ]:
preview_idx = 0 # change to view another file
md_files = sorted(p for p in MD_OUTPUT_DIR.glob('*.md') if p.is_file())
assert md_files, "No .md files produced."
target = md_files[preview_idx]
print(f"# Preview of: {target.name}\n" + "=" * 60)
content = target.read_text(encoding='utf-8')
print(content[:3000])
if len(content) > 3000:
print(f"\n… ({len(content)-3000:,} more chars)")In [ ]:
import shutil
from google.colab import files as colab_files
ZIP_PATH = Path('/content/markdown_for_openwebui.zip')
if ZIP_PATH.exists():
ZIP_PATH.unlink()
shutil.make_archive('/content/markdown_for_openwebui', 'zip',
root_dir=str(MD_OUTPUT_DIR))
print(f"Created {ZIP_PATH} ({ZIP_PATH.stat().st_size:,} bytes)")
colab_files.download(str(ZIP_PATH))In [ ]:
RUN_DOWNLOADER = False # <-- flip to True to actually run the downloader
if RUN_DOWNLOADER:
import requests, csv, hashlib, time
from pathlib import Path
# Make sure downloads.csv has a header if it's new.
if not DOWNLOADS_CSV.exists():
with open(DOWNLOADS_CSV, 'w', newline='') as f:
csv.DictWriter(f, fieldnames=DOWNLOADS_COLS).writeheader()
# Load already-downloaded URLs
already = set()
with open(DOWNLOADS_CSV, 'r', newline='') as f:
for r in csv.DictReader(f):
already.add(r['url'])
if not SOURCES_PATH.exists():
print("No sources.csv found. Run the URL extraction cell first.")
else:
with open(SOURCES_PATH, 'r', newline='') as f:
source_rows = list(csv.DictReader(f))
unique_urls = sorted({r['url'] for r in source_rows})
todo = [u for u in unique_urls if u not in already]
print(f"{len(unique_urls)} unique URLs in sources.csv; "
f"{len(already)} already downloaded; "
f"will fetch {len(todo)}.")
fetched = 0
failed = 0
for url in unique_urls:
if url in already:
continue
try:
# HEAD first to get content-type and final URL.
h = requests.head(url, allow_redirects=True, timeout=15,
headers={'User-Agent': 'Mozilla/5.0'})
ctype = h.headers.get('Content-Type', '').lower()
if 'pdf' in ctype or url.lower().endswith('.pdf'):
kind, ext = 'pdf', 'pdf'
elif 'html' in ctype:
kind, ext = 'html', 'html'
else:
kind, ext = 'other', 'bin'
if not (200 <= h.status_code < 300):
with open(DOWNLOADS_CSV, 'a', newline='') as f:
csv.DictWriter(f, fieldnames=DOWNLOADS_COLS).writerow({
'url': url, 'sha256_of_bytes': '',
'saved_path': '', 'http_status': h.status_code,
'content_type': ctype, 'downloaded_at': _now_iso()})
print(f"✗ {url} → HTTP {h.status_code} (skipped)")
failed += 1
time.sleep(RATE_LIMIT_SEC)
continue
# Fetch the actual bytes.
r = requests.get(url, allow_redirects=True, timeout=30,
headers={'User-Agent': 'Mozilla/5.0'})
if not (200 <= r.status_code < 300):
with open(DOWNLOADS_CSV, 'a', newline='') as f:
csv.DictWriter(f, fieldnames=DOWNLOADS_COLS).writerow({
'url': url, 'sha256_of_bytes': '',
'saved_path': '', 'http_status': r.status_code,
'content_type': r.headers.get('Content-Type', ''),
'downloaded_at': _now_iso()})
print(f"✗ {url} → HTTP {r.status_code}")
failed += 1
time.sleep(RATE_LIMIT_SEC)
continue
body_sha = hashlib.sha256(r.content).hexdigest()
saved = DOWNLOADS_DIR / kind / f"{body_sha}.{ext}"
saved.write_bytes(r.content)
rel = str(saved.relative_to(DRIVE_ROOT))
with open(DOWNLOADS_CSV, 'a', newline='') as f:
csv.DictWriter(f, fieldnames=DOWNLOADS_COLS).writerow({
'url': url, 'sha256_of_bytes': body_sha,
'saved_path': rel, 'http_status': r.status_code,
'content_type': r.headers.get('Content-Type', ''),
'downloaded_at': _now_iso()})
fetched += 1
print(f"✓ {url} → {rel} ({len(r.content):,} bytes)")
time.sleep(RATE_LIMIT_SEC)
except Exception as e:
print(f"✗ {url} → {type(e).__name__}: {e}")
failed += 1
time.sleep(RATE_LIMIT_SEC)
print(f"\nDone. Fetched {fetched} new, failed/skipped {failed}.")
else:
print("Downloader is OFF. Set RUN_DOWNLOADER = True above to run.")