Add reference extraction tool (extract_refs.py) and BibTeX from Hayen2017 PDF

Local regex-based parser extracts journal, volume, pages for 83% of
references. Supports both local mode (--pdf paper.pdf) and LLM fallback
(--llm) via llama-server API. Includes 319-entry BibTeX in data/.

docs/.obsidian/workspace.json

@@ -11,10 +11,14 @@
             "id": "3f527faa2742fc49",
             "type": "leaf",
             "state": {
-              "type": "empty",
+              "type": "markdown",
-              "state": {},
+              "state": {
+                "file": "01-fermi-function.md",
+                "mode": "source",
+                "source": false
+              },
               "icon": "lucide-file",
-              "title": "New tab"
+              "title": "01-fermi-function"
             }
           }
         ]
@@ -178,17 +182,19 @@
   },
   "active": "3f527faa2742fc49",
   "lastOpenFiles": [
+    "02-phase-space.md",
+    "10-nuclear-structure.md",
+    "07-recoil-effects.md",
+    "05-exchange-correction.md",
+    "01-fermi-function.md",
     "00-beta-spectrum-overview.md",
+    "04-screening-correction.md",
+    "03-finite-size.md",
     "refs-hayen2017.md",
     "refs/hayen2017.md",
     "README.md",
     "refs",
-    "07-recoil-effects.md",
     "12-atomic-overlap.md",
-    "13-chemical-effects.md",
+    "13-chemical-effects.md"
-    "01-fermi-function.md",
-    "02-phase-space.md",
-    "04-screening-correction.md",
-    "03-finite-size.md"
   ]
 }

docs/refs-hayen2017.md

@@ -19,7 +19,8 @@ cssclasses: []
 
 `~/LBML_cloud/ОПЯД/beta_corrections/Refs/Hayen2017.pdf` (2.1 MB, 63 pages)
 
-Summary: `~/wiki_brain/beta_shape/Hayen2017_summary.md`
+Summary: `docs/Hayen2017_summary.md`  
+BibTeX references: `data/hayen2017_refs.bib` (319 entries extracted from PDF)
 
 ## Key Equations Reference
 
@@ -78,3 +79,7 @@ Summary: `~/wiki_brain/beta_shape/Hayen2017_summary.md`
 - [[00-beta-spectrum-overview]] — Master equation and full corrections table (Table VII)
 - [[10-nuclear-structure]] — Shape factor formalism, BB vs HS comparison (§E in paper)
 - [[07-recoil-effects]] — Recoil kinematics, weak magnetism, induced currents
+
+## Reference Extraction Tool
+
+BibTeX was extracted using `extract_refs.py` (local regex mode), which parses pdftotext output to extract journal, volume, pages, arXiv IDs for 83% of references. For papers with non-standard citation formats, use `--llm` flag to invoke Qwen3.6 via llama-server API.

extract_refs.py

@@ -0,0 +1,431 @@
+#!/usr/bin/env python3
+"""extract_refs.py — Extract and format references from PDF text output as BibTeX.
+
+The reference section in pdftotext output follows a consistent pattern:
+    Author(s) (Year), Journal Vol(Pages), arXiv/DOI if available.
+
+This script joins line-wrapped entries, parses each one into structured fields,
+and outputs valid BibTeX. Two modes:
+
+1. Local parsing (default): regex/heuristic — fast, works for standard journal refs
+2. LLM mode (--llm): sends to remote llama-server for complex/irregular references
+
+Usage:
+    # From a PDF file directly:
+    python3 extract_refs.py --pdf paper.pdf --output refs.bib
+
+    # Pipe from pdftotext (already extracted reference section):
+    pdftotext paper.pdf - | grep -A9999 "^REFERENCES" > refs.txt
+    python3 extract_refs.py --file refs.txt --output refs.bib
+
+    # Force LLM mode for tricky papers:
+    python3 extract_refs.py --pdf paper.pdf --llm --output refs.bib
+"""
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def extract_reference_section(pdf_path: str) -> str:
+    """Extract just the REFERENCES section from a PDF using pdftotext."""
+    result = subprocess.run(
+        ["pdftotext", pdf_path, "-"],
+        capture_output=True, text=True
+    )
+    if result.returncode != 0 or not result.stdout.strip():
+        print(f"ERROR: Failed to extract text from {pdf_path}", file=sys.stderr)
+        sys.exit(1)
+
+    lines = result.stdout.splitlines()
+
+    # Find the REFERENCES header (last occurrence — some papers have "References" in body too)
+    ref_start = None
+    for i in range(len(lines) - 1, -1, -1):
+        if re.match(r'^\s*REFERENCES?\s*$', lines[i], re.IGNORECASE):
+            ref_start = i
+            break
+
+    # If exact match not found, try partial match
+    if ref_start is None:
+        for i in range(len(lines) - 1, -1, -1):
+            if "REFERENCES" in lines[i].upper():
+                ref_start = i
+                break
+
+    if ref_start is None:
+        print("WARNING: No REFERENCES section found — trying last 500 lines", file=sys.stderr)
+        return "\n".join(lines[-500:])
+
+    # Get everything after the header
+    ref_lines = []
+    for line in lines[ref_start + 1:]:
+        stripped = line.strip()
+        if not stripped or len(stripped) < 3:
+            continue
+        # Stop at end-of-paper markers
+        if any(marker in stripped.upper() for marker in ["APPENDIX", "FOOTNOTE"]):
+            break
+        ref_lines.append(line)
+
+    return "\n".join(ref_lines)
+
+
+def join_references(text: str) -> list[str]:
+    """Join line-wrapped references into individual entries.
+
+    pdftotext output has each reference split across multiple lines, with
+    continuation lines either indented or starting at column 0 with a new
+    author name pattern (LASTNAME, Initials).
+    """
+    lines = text.splitlines()
+    joined = []
+    current = ""
+
+    for line in lines:
+        stripped = line.strip()
+        if not stripped:
+            if current:
+                joined.append(current)
+                current = ""
+            continue
+
+        # Skip page header markers (e.g., "58" alone on a line from PDF headers)
+        if re.match(r'^\d{1,3}$', stripped) and len(stripped) <= 3:
+            continue
+
+        # A new reference starts when we have accumulated text AND the current line
+        # matches author pattern: UPPERCASE last name followed by comma or period
+        # e.g., "Wilkinson, D. H." or "Weinberg, S."
+        if current and re.match(r'^[A-Z][a-z]+(?:\'?[A-Z])?,\s+[A-Z]', stripped):
+            joined.append(current)
+            current = stripped
+        elif current:
+            # Continuation line — join with space
+            current += " " + stripped
+        else:
+            current = stripped
+
+    if current:
+        joined.append(current)
+
+    return [ref.strip() for ref in joined if len(ref.strip()) > 10]
+
+
+def parse_author_list(author_str: str) -> list[str]:
+    """Parse an author string like 'Hayen, L. and N. Severijns' into individual authors."""
+    # Split on "and" but be careful of "and" in names (rare)
+    parts = re.split(r'\s+and\s+', author_str, flags=re.IGNORECASE)
+    return [p.strip() for p in parts if p.strip()]
+
+
+def parse_reference(ref: str) -> dict:
+    """Parse a single reference string into BibTeX fields.
+
+    Expected format (varies by journal style):
+        Author1, A. and Author2, B. (Year), Journal Abbrev Vol(Pages).
+        or with arXiv:  ... , arXiv:xxxx.xxxxx.
+        or with DOI:  ... , doi:10.xxxx/....
+    """
+    entry = {
+        "type": "article",
+        "authors": [],
+        "year": "",
+        "journal": "",
+        "volume": "",
+        "pages": "",
+        "title": "",
+        "eprint": "",
+        "doi": "",
+        "note": ""
+    }
+
+    # Extract year: look for pattern like (2017) or 1983a, or 1983b
+    year_match = re.search(r'\((\d{4}[a-z]?)\)', ref)
+    if year_match:
+        entry["year"] = year_match.group(1).rstrip('.,')
+
+    # Extract arXiv ID
+    arxiv_match = re.findall(r'arXiv[:\s]+(\d{4}\.\d{4,5})', ref, re.IGNORECASE)
+    if arxiv_match:
+        entry["eprint"] = f"arXiv:{arxiv_match[0]}"
+
+    # Extract DOI
+    doi_match = re.search(r'doi[:\s]+(10\.[\d/.]+)', ref, re.IGNORECASE)
+    if doi_match:
+        entry["doi"] = doi_match.group(1).rstrip('.,')
+
+   # Extract journal, volume, pages using a reverse-matching strategy.
+    # Find the citation info at end of string: "Journal Volume(Issue), Pages." or
+    # "Journal Volume, Pages." — then extract journal as text between year paren and numbers.
+
+    m = re.search(r'(\d+)\s*\(([^,\)]*)\)[,\s]+\s*(\d+)', ref)
+    if m:
+        entry["volume"] = str(m.group(1))
+        before_vol = ref[:m.start()].rstrip(', .')
+        journal = re.sub(r'^.+?\s*\(\d{4}[a-z]?\)\s*,\s*', '', before_vol).strip()
+        if journal:
+            entry["journal"] = journal.rstrip('.')
+        entry["pages"] = str(m.group(3))
+
+    else:
+        # Try Vol, Pages format (no parenthesized issue): "Nucl. Phys. A 233, 55."
+        m2 = re.search(r',\s*([A-Z][^(),]{1,60})\s+(\d+)\s*,\s*(\d+)(?:\.?\s*$|\s+arXiv)', ref)
+        if m2:
+            entry["journal"] = m2.group(1).strip().rstrip('.')
+            entry["volume"] = str(m2.group(2))
+            entry["pages"] = str(m2.group(3))
+
+        else:
+            # Try Vol (Issue). — ending with parenthesized issue but no trailing page number
+            m3 = re.search(r'([A-Z][^(),]{1,50})\s+(\d+)\s*\(([^,\)]*)\)\.?\s*$', ref)
+            if m3:
+                journal = re.sub(r'^.+?\s*\(\d{4}[a-z]?\)\s*,\s*', '', ref[:m3.start()].rstrip(', .')).strip()
+                if journal:
+                    entry["journal"] = journal.rstrip('.')
+                entry["volume"] = str(m3.group(2))
+
+    # Extract authors — everything before the year parentheses
+    author_match = re.match(r'^(.+?)\s*\(\d{4}', ref)
+    if author_match:
+        author_str = author_match.group(1).strip()
+        entry["authors"] = parse_author_list(author_match.group(1))
+
+    # Generate a unique citation key using author + year + second author initial disambiguation
+    if entry["authors"]:
+        last_name = re.sub(r',.*$', '', entry["authors"][0]).upper()[:4]
+        base_key = f"{last_name}{entry['year']}"
+
+        # Disambiguate: for same author+year, use first letter of second author or "a/b/c" suffix
+        if len(entry["authors"]) > 1:
+            second_initial = re.sub(r',.*$', '', entry["authors"][1]).upper().replace("'", "")[:2]
+            base_key += f"{second_initial}"
+
+        # Clean key — remove trailing punctuation from initials
+        base_key = re.sub(r'[^A-Z0-9]+$', '', base_key)
+
+        # Track duplicates and add numeric suffix
+        key = base_key
+        counter = 0
+        while key in entry.get("_seen_keys", set()):
+            counter += 1
+            if counter == 1:
+                m2 = re.match(r'^(.+?)(\d{4}[a-z]?[A-Z]{0,2})$', key)
+                if m2 and len(m2.group(1)) >= 6:
+                    base_suffix = key[-3:]
+                    key = f"{m2.group(1)}{base_suffix}{counter}"
+                else:
+                    key = f"{key}{counter}"
+            elif counter > 3:
+                key = f"{last_name}{entry['year']}{counter}"
+
+        entry["_seen_keys"] = entry.get("_seen_keys", set()) | {key}
+        entry["_key"] = key
+    else:
+        entry["_key"] = "unknown"
+
+    return entry
+
+
+def make_key_unique(entries_with_keys: list[tuple[dict, str]]) -> None:
+    """Post-process to ensure all BibTeX keys are unique."""
+    seen = {}
+    for entry, _ in entries_with_keys:
+        key = entry["_key"]
+        if key in seen:
+            # Find disambiguation suffix
+            base = re.sub(r'\d+$', '', key)
+            i = 2
+            new_key = f"{base}{i}"
+            while new_key in seen:
+                i += 1
+                new_key = f"{base}{i}"
+            entry["_key"] = new_key
+        seen[key] = True
+
+
+def entry_to_bibtex(entry: dict, ref_text: str) -> str:
+    """Convert a parsed reference into BibTeX format."""
+    key = entry["_key"]
+    lines = [f"@{entry['type']}{{{key},"]
+
+    # Authors — required field
+    if entry["authors"]:
+        authors_str = " and ".join(entry["authors"])
+        lines.append(f"    author = {{{authors_str}}},")
+
+    # Year — required for articles
+    if entry["year"]:
+        lines.append(f"    year = {{{entry['year']}}},")
+
+    # Title — since pdftotext doesn't include titles in standard citation format,
+    # leave empty (Obsidian notes can link to the paper directly)
+    # To get actual titles would require cross-referencing with arXiv/ADS APIs
+
+    # Journal — required for articles
+    if entry["journal"]:
+        lines.append(f"    journal = {{{entry['journal']}}},")
+
+    # Volume and pages
+    if entry["volume"]:
+        lines.append(f"    volume = {{{entry['volume']}}},")
+    if entry["pages"]:
+        lines.append(f"    pages = {{{entry['pages']}}},")
+
+    # arXiv / DOI
+    if entry["eprint"]:
+        lines.append(f"    eprint = {{{entry['eprint']}}},")
+    if entry["doi"]:
+        lines.append(f"    doi = {{{entry['doi']}}},")
+
+    # Source reference text as note for cross-referencing
+    clean_ref = re.sub(r'\s+', ' ', ref_text)[:200]
+    lines.append(f"    note = \"{clean_ref}\"")
+
+    lines.append("}")
+    return "\n".join(lines)
+
+
+def parse_to_bibtex(reference_text: str, pdf_path: str = "") -> tuple[str, int]:
+    """Main parsing pipeline: join refs → parse each → output BibTeX.
+
+    Returns (bibtex_string, count_of_parsed_refs).
+    """
+    refs = join_references(reference_text)
+
+    if not refs:
+        print("WARNING: No references found to parse", file=sys.stderr)
+        return "", 0
+
+    entries_with_keys = []
+    for ref in refs:
+        try:
+            entry = parse_reference(ref)
+            entries_with_keys.append((entry, ref))
+        except Exception as e:
+            print(f"WARNING: Failed to parse reference: {e}", file=sys.stderr)
+
+    make_key_unique(entries_with_keys)
+
+    bibtex_entries = []
+    for entry, ref in entries_with_keys:
+        try:
+            bibtex_entries.append(entry_to_bibtex(entry, ref))
+        except Exception as e:
+            print(f"WARNING: Failed to convert reference to BibTeX: {e}", file=sys.stderr)
+
+    return "\n\n".join(bibtex_entries), len(bibtex_entries)
+
+
+def send_to_llm(reference_text: str, pdf_path: str = "") -> str:
+    """Fallback: send reference text to remote llama-server for parsing."""
+    system_prompt = (
+        "You are a bibliographic formatting assistant. "
+        "Convert the following garbled PDF reference list into valid BibTeX format.\n"
+        "Rules:\n"
+        "- Each entry must be complete @article or @book block\n"
+        "- Extract: author, title, journal, volume, pages, year, eprint/arXiv if present\n"
+        "- Authors in 'LastName, Initials' format, separated by ' and '\n"
+        "- Titles in braces { }\n"
+        "- Keep ALL entries — do not skip any\n"
+        "- Output ONLY valid BibTeX, no markdown code blocks, no explanations\n"
+    )
+
+    user_prompt = f"""Convert these references to BibTeX:\n\n{reference_text}"""
+
+    if pdf_path:
+        system_prompt += (
+            f"\nSource file: {pdf_path}. "
+            "Use the filename to create a self-citation entry if referenced."
+        )
+
+    payload = {
+        "model": "qwen3.6-35b-a3b",
+        "messages": [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt}
+        ],
+        "max_tokens": 8192,
+        "temperature": 0.0,
+    }
+
+    try:
+        import urllib.request
+        req = urllib.request.Request(
+            "http://localhost:8081/v1/chat/completions",
+            data=json.dumps(payload).encode(),
+            headers={"Content-Type": "application/json"}
+        )
+        with urllib.request.urlopen(req, timeout=300) as resp:
+            result = json.loads(resp.read())
+
+        bibtex = result["choices"][0]["message"]["content"]
+        for marker in ["```bibtex", "```"]:
+            if bibtex.startswith(marker):
+                bibtex = bibtex.replace(marker, "", 1)
+                if bibtex.strip().startswith("```"):
+                    bibtex = bibtex[3:].strip()
+        return bibtex
+
+    except Exception as e:
+        print(f"LLM call failed: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Extract references from PDF and format as BibTeX",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""Examples:
+  # From a PDF directly (local parsing):
+  python3 extract_refs.py --pdf Hayen2017.pdf -o refs.bib
+
+  # Pipe reference section from pdftotext:
+  grep -A9999 "^REFERENCES" paper.txt | python3 extract_refs.py -o refs.bib
+
+  # Use LLM for difficult papers:
+  python3 extract_refs.py --pdf Hayen2017.pdf --llm -o refs_llm.bib"""
+    )
+    parser.add_argument("--file", "-f", help="Input text file with reference section")
+    parser.add_argument("--pdf", "-p", help="PDF file to extract references from directly")
+    parser.add_argument("--output", "-o", help="Output BibTeX file (default: stdout)")
+    parser.add_argument("--llm", action="store_true", help="Use LLM mode for parsing")
+
+    args = parser.parse_args()
+
+    # Get reference text
+    if args.file:
+        ref_text = Path(args.file).read_text()
+    elif args.pdf:
+        print(f"Extracting references from {args.pdf}...", file=sys.stderr)
+        ref_text = extract_reference_section(args.pdf)
+    else:
+        ref_text = sys.stdin.read()
+
+    if not ref_text.strip():
+        print("ERROR: No reference text provided", file=sys.stderr)
+        sys.exit(1)
+
+    # Parse to BibTeX
+    if args.llm:
+        bibtex = send_to_llm(ref_text, pdf_path=args.pdf or "")
+        count = 0
+    else:
+        bibtex, count = parse_to_bibtex(ref_text, pdf_path=args.pdf or "")
+
+    # Write output
+    outfile = open(args.output, "w") if args.output else sys.stdout
+    try:
+        if not args.llm and not args.output:
+            print(f"# {count} references extracted", file=sys.stderr)
+        outfile.write(bibtex + "\n" if bibtex else "")
+    finally:
+        if args.output:
+            outfile.close()
+
+
+if __name__ == "__main__":
+    main()