unzhakov/beta_shape_pnpi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add reference extraction tool (extract_refs.py) and BibTeX from Hayen2017 PDF

Browse Source 
Local regex-based parser extracts journal, volume, pages for 83% of
references. Supports both local mode (--pdf paper.pdf) and LLM fallback
(--llm) via llama-server API. Includes 319-entry BibTeX in data/.
This commit is contained in:
unzhakov
2026-04-27 11:03:06 +03:00
parent bce6e63663
commit 85134396d6
4 changed files with 3153 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/hayen2017_refs.bib
+2701
View File
File diff suppressed because it is too large Load Diff
docs/.obsidian/workspace.json
Vendored
+15 -9
View File
@@ -11,10 +11,14 @@
"id": "3f527faa2742fc49",
"type": "leaf",
"state": {
"type": "empty",
"state": {},
"type": "markdown",
"state": {
"file": "01-fermi-function.md",
"mode": "source",
"source": false
},
"icon": "lucide-file",
"title": "New tab"
"title": "01-fermi-function"
}
}
]
@@ -178,17 +182,19 @@
},
"active": "3f527faa2742fc49",
"lastOpenFiles": [
"02-phase-space.md",
"10-nuclear-structure.md",
"07-recoil-effects.md",
"05-exchange-correction.md",
"01-fermi-function.md",
"00-beta-spectrum-overview.md",
"04-screening-correction.md",
"03-finite-size.md",
"refs-hayen2017.md",
"refs/hayen2017.md",
"README.md",
"refs",
"07-recoil-effects.md",
"12-atomic-overlap.md",
"13-chemical-effects.md",
"01-fermi-function.md",
"02-phase-space.md",
"04-screening-correction.md",
"03-finite-size.md"
"13-chemical-effects.md"
]
}
docs/refs-hayen2017.md
+6 -1
View File
@@ -19,7 +19,8 @@ cssclasses: []
`~/LBML_cloud/ОПЯД/beta_corrections/Refs/Hayen2017.pdf` (2.1 MB, 63 pages)
Summary: `~/wiki_brain/beta_shape/Hayen2017_summary.md`
Summary: `docs/Hayen2017_summary.md`
BibTeX references: `data/hayen2017_refs.bib` (319 entries extracted from PDF)
## Key Equations Reference
@@ -78,3 +79,7 @@ Summary: `~/wiki_brain/beta_shape/Hayen2017_summary.md`
- [[00-beta-spectrum-overview]] — Master equation and full corrections table (Table VII)
- [[10-nuclear-structure]] — Shape factor formalism, BB vs HS comparison (§E in paper)
- [[07-recoil-effects]] — Recoil kinematics, weak magnetism, induced currents
## Reference Extraction Tool
BibTeX was extracted using `extract_refs.py` (local regex mode), which parses pdftotext output to extract journal, volume, pages, arXiv IDs for 83% of references. For papers with non-standard citation formats, use `--llm` flag to invoke Qwen3.6 via llama-server API.
extract_refs.py
+431
View File
@@ -0,0 +1,431 @@
#!/usr/bin/env python3
"""extract_refs.py — Extract and format references from PDF text output as BibTeX.
The reference section in pdftotext output follows a consistent pattern:
Author(s) (Year), Journal Vol(Pages), arXiv/DOI if available.
This script joins line-wrapped entries, parses each one into structured fields,
and outputs valid BibTeX. Two modes:
1. Local parsing (default): regex/heuristic — fast, works for standard journal refs
2. LLM mode (--llm): sends to remote llama-server for complex/irregular references
Usage:
# From a PDF file directly:
python3 extract_refs.py --pdf paper.pdf --output refs.bib
# Pipe from pdftotext (already extracted reference section):
pdftotext paper.pdf - | grep -A9999 "^REFERENCES" > refs.txt
python3 extract_refs.py --file refs.txt --output refs.bib
# Force LLM mode for tricky papers:
python3 extract_refs.py --pdf paper.pdf --llm --output refs.bib
"""
import argparse
import json
import re
import subprocess
import sys
from pathlib import Path
def extract_reference_section(pdf_path: str) -> str:
"""Extract just the REFERENCES section from a PDF using pdftotext."""
result = subprocess.run(
["pdftotext", pdf_path, "-"],
capture_output=True, text=True
)
if result.returncode != 0 or not result.stdout.strip():
print(f"ERROR: Failed to extract text from {pdf_path}", file=sys.stderr)
sys.exit(1)
lines = result.stdout.splitlines()
# Find the REFERENCES header (last occurrence — some papers have "References" in body too)
ref_start = None
for i in range(len(lines) - 1, -1, -1):
if re.match(r'^\s*REFERENCES?\s*$', lines[i], re.IGNORECASE):
ref_start = i
break
# If exact match not found, try partial match
if ref_start is None:
for i in range(len(lines) - 1, -1, -1):
if "REFERENCES" in lines[i].upper():
ref_start = i
break
if ref_start is None:
print("WARNING: No REFERENCES section found — trying last 500 lines", file=sys.stderr)
return "\n".join(lines[-500:])
# Get everything after the header
ref_lines = []
for line in lines[ref_start + 1:]:
stripped = line.strip()
if not stripped or len(stripped) < 3:
continue
# Stop at end-of-paper markers
if any(marker in stripped.upper() for marker in ["APPENDIX", "FOOTNOTE"]):
break
ref_lines.append(line)
return "\n".join(ref_lines)
def join_references(text: str) -> list[str]:
"""Join line-wrapped references into individual entries.
pdftotext output has each reference split across multiple lines, with
continuation lines either indented or starting at column 0 with a new
author name pattern (LASTNAME, Initials).
"""
lines = text.splitlines()
joined = []
current = ""
for line in lines:
stripped = line.strip()
if not stripped:
if current:
joined.append(current)
current = ""
continue
# Skip page header markers (e.g., "58" alone on a line from PDF headers)
if re.match(r'^\d{1,3}$', stripped) and len(stripped) <= 3:
continue
# A new reference starts when we have accumulated text AND the current line
# matches author pattern: UPPERCASE last name followed by comma or period
# e.g., "Wilkinson, D. H." or "Weinberg, S."
if current and re.match(r'^[A-Z][a-z]+(?:\'?[A-Z])?,\s+[A-Z]', stripped):
joined.append(current)
current = stripped
elif current:
# Continuation line — join with space
current += " " + stripped
else:
current = stripped
if current:
joined.append(current)
return [ref.strip() for ref in joined if len(ref.strip()) > 10]
def parse_author_list(author_str: str) -> list[str]:
"""Parse an author string like 'Hayen, L. and N. Severijns' into individual authors."""
# Split on "and" but be careful of "and" in names (rare)
parts = re.split(r'\s+and\s+', author_str, flags=re.IGNORECASE)
return [p.strip() for p in parts if p.strip()]
def parse_reference(ref: str) -> dict:
"""Parse a single reference string into BibTeX fields.
Expected format (varies by journal style):
Author1, A. and Author2, B. (Year), Journal Abbrev Vol(Pages).
or with arXiv: ... , arXiv:xxxx.xxxxx.
or with DOI: ... , doi:10.xxxx/....
"""
entry = {
"type": "article",
"authors": [],
"year": "",
"journal": "",
"volume": "",
"pages": "",
"title": "",
"eprint": "",
"doi": "",
"note": ""
}
# Extract year: look for pattern like (2017) or 1983a, or 1983b
year_match = re.search(r'\((\d{4}[a-z]?)\)', ref)
if year_match:
entry["year"] = year_match.group(1).rstrip('.,')
# Extract arXiv ID
arxiv_match = re.findall(r'arXiv[:\s]+(\d{4}\.\d{4,5})', ref, re.IGNORECASE)
if arxiv_match:
entry["eprint"] = f"arXiv:{arxiv_match[0]}"
# Extract DOI
doi_match = re.search(r'doi[:\s]+(10\.[\d/.]+)', ref, re.IGNORECASE)
if doi_match:
entry["doi"] = doi_match.group(1).rstrip('.,')
# Extract journal, volume, pages using a reverse-matching strategy.
# Find the citation info at end of string: "Journal Volume(Issue), Pages." or
# "Journal Volume, Pages." — then extract journal as text between year paren and numbers.
m = re.search(r'(\d+)\s*\(([^,\)]*)\)[,\s]+\s*(\d+)', ref)
if m:
entry["volume"] = str(m.group(1))
before_vol = ref[:m.start()].rstrip(', .')
journal = re.sub(r'^.+?\s*\(\d{4}[a-z]?\)\s*,\s*', '', before_vol).strip()
if journal:
entry["journal"] = journal.rstrip('.')
entry["pages"] = str(m.group(3))
else:
# Try Vol, Pages format (no parenthesized issue): "Nucl. Phys. A 233, 55."
m2 = re.search(r',\s*([A-Z][^(),]{1,60})\s+(\d+)\s*,\s*(\d+)(?:\.?\s*$|\s+arXiv)', ref)
if m2:
entry["journal"] = m2.group(1).strip().rstrip('.')
entry["volume"] = str(m2.group(2))
entry["pages"] = str(m2.group(3))
else:
# Try Vol (Issue). — ending with parenthesized issue but no trailing page number
m3 = re.search(r'([A-Z][^(),]{1,50})\s+(\d+)\s*\(([^,\)]*)\)\.?\s*$', ref)
if m3:
journal = re.sub(r'^.+?\s*\(\d{4}[a-z]?\)\s*,\s*', '', ref[:m3.start()].rstrip(', .')).strip()
if journal:
entry["journal"] = journal.rstrip('.')
entry["volume"] = str(m3.group(2))
# Extract authors — everything before the year parentheses
author_match = re.match(r'^(.+?)\s*\(\d{4}', ref)
if author_match:
author_str = author_match.group(1).strip()
entry["authors"] = parse_author_list(author_match.group(1))
# Generate a unique citation key using author + year + second author initial disambiguation
if entry["authors"]:
last_name = re.sub(r',.*$', '', entry["authors"][0]).upper()[:4]
base_key = f"{last_name}{entry['year']}"
# Disambiguate: for same author+year, use first letter of second author or "a/b/c" suffix
if len(entry["authors"]) > 1:
second_initial = re.sub(r',.*$', '', entry["authors"][1]).upper().replace("'", "")[:2]
base_key += f"{second_initial}"
# Clean key — remove trailing punctuation from initials
base_key = re.sub(r'[^A-Z0-9]+$', '', base_key)
# Track duplicates and add numeric suffix
key = base_key
counter = 0
while key in entry.get("_seen_keys", set()):
counter += 1
if counter == 1:
m2 = re.match(r'^(.+?)(\d{4}[a-z]?[A-Z]{0,2})$', key)
if m2 and len(m2.group(1)) >= 6:
base_suffix = key[-3:]
key = f"{m2.group(1)}{base_suffix}{counter}"
else:
key = f"{key}{counter}"
elif counter > 3:
key = f"{last_name}{entry['year']}{counter}"
entry["_seen_keys"] = entry.get("_seen_keys", set()) | {key}
entry["_key"] = key
else:
entry["_key"] = "unknown"
return entry
def make_key_unique(entries_with_keys: list[tuple[dict, str]]) -> None:
"""Post-process to ensure all BibTeX keys are unique."""
seen = {}
for entry, _ in entries_with_keys:
key = entry["_key"]
if key in seen:
# Find disambiguation suffix
base = re.sub(r'\d+$', '', key)
i = 2
new_key = f"{base}{i}"
while new_key in seen:
i += 1
new_key = f"{base}{i}"
entry["_key"] = new_key
seen[key] = True
def entry_to_bibtex(entry: dict, ref_text: str) -> str:
"""Convert a parsed reference into BibTeX format."""
key = entry["_key"]
lines = [f"@{entry['type']}{{{key},"]
# Authors — required field
if entry["authors"]:
authors_str = " and ".join(entry["authors"])
lines.append(f" author = {{{authors_str}}},")
# Year — required for articles
if entry["year"]:
lines.append(f" year = {{{entry['year']}}},")
# Title — since pdftotext doesn't include titles in standard citation format,
# leave empty (Obsidian notes can link to the paper directly)
# To get actual titles would require cross-referencing with arXiv/ADS APIs
# Journal — required for articles
if entry["journal"]:
lines.append(f" journal = {{{entry['journal']}}},")
# Volume and pages
if entry["volume"]:
lines.append(f" volume = {{{entry['volume']}}},")
if entry["pages"]:
lines.append(f" pages = {{{entry['pages']}}},")
# arXiv / DOI
if entry["eprint"]:
lines.append(f" eprint = {{{entry['eprint']}}},")
if entry["doi"]:
lines.append(f" doi = {{{entry['doi']}}},")
# Source reference text as note for cross-referencing
clean_ref = re.sub(r'\s+', ' ', ref_text)[:200]
lines.append(f" note = \"{clean_ref}\"")
lines.append("}")
return "\n".join(lines)
def parse_to_bibtex(reference_text: str, pdf_path: str = "") -> tuple[str, int]:
"""Main parsing pipeline: join refs → parse each → output BibTeX.
Returns (bibtex_string, count_of_parsed_refs).
"""
refs = join_references(reference_text)
if not refs:
print("WARNING: No references found to parse", file=sys.stderr)
return "", 0
entries_with_keys = []
for ref in refs:
try:
entry = parse_reference(ref)
entries_with_keys.append((entry, ref))
except Exception as e:
print(f"WARNING: Failed to parse reference: {e}", file=sys.stderr)
make_key_unique(entries_with_keys)
bibtex_entries = []
for entry, ref in entries_with_keys:
try:
bibtex_entries.append(entry_to_bibtex(entry, ref))
except Exception as e:
print(f"WARNING: Failed to convert reference to BibTeX: {e}", file=sys.stderr)
return "\n\n".join(bibtex_entries), len(bibtex_entries)
def send_to_llm(reference_text: str, pdf_path: str = "") -> str:
"""Fallback: send reference text to remote llama-server for parsing."""
system_prompt = (
"You are a bibliographic formatting assistant. "
"Convert the following garbled PDF reference list into valid BibTeX format.\n"
"Rules:\n"
"- Each entry must be complete @article or @book block\n"
"- Extract: author, title, journal, volume, pages, year, eprint/arXiv if present\n"
"- Authors in 'LastName, Initials' format, separated by ' and '\n"
"- Titles in braces { }\n"
"- Keep ALL entries — do not skip any\n"
"- Output ONLY valid BibTeX, no markdown code blocks, no explanations\n"
)
user_prompt = f"""Convert these references to BibTeX:\n\n{reference_text}"""
if pdf_path:
system_prompt += (
f"\nSource file: {pdf_path}. "
"Use the filename to create a self-citation entry if referenced."
)
payload = {
"model": "qwen3.6-35b-a3b",
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
],
"max_tokens": 8192,
"temperature": 0.0,
}
try:
import urllib.request
req = urllib.request.Request(
"http://localhost:8081/v1/chat/completions",
data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"}
)
with urllib.request.urlopen(req, timeout=300) as resp:
result = json.loads(resp.read())
bibtex = result["choices"][0]["message"]["content"]
for marker in ["```bibtex", "```"]:
if bibtex.startswith(marker):
bibtex = bibtex.replace(marker, "", 1)
if bibtex.strip().startswith("```"):
bibtex = bibtex[3:].strip()
return bibtex
except Exception as e:
print(f"LLM call failed: {e}", file=sys.stderr)
sys.exit(1)
def main():
parser = argparse.ArgumentParser(
description="Extract references from PDF and format as BibTeX",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""Examples:
# From a PDF directly (local parsing):
python3 extract_refs.py --pdf Hayen2017.pdf -o refs.bib
# Pipe reference section from pdftotext:
grep -A9999 "^REFERENCES" paper.txt | python3 extract_refs.py -o refs.bib
# Use LLM for difficult papers:
python3 extract_refs.py --pdf Hayen2017.pdf --llm -o refs_llm.bib"""
)
parser.add_argument("--file", "-f", help="Input text file with reference section")
parser.add_argument("--pdf", "-p", help="PDF file to extract references from directly")
parser.add_argument("--output", "-o", help="Output BibTeX file (default: stdout)")
parser.add_argument("--llm", action="store_true", help="Use LLM mode for parsing")
args = parser.parse_args()
# Get reference text
if args.file:
ref_text = Path(args.file).read_text()
elif args.pdf:
print(f"Extracting references from {args.pdf}...", file=sys.stderr)
ref_text = extract_reference_section(args.pdf)
else:
ref_text = sys.stdin.read()
if not ref_text.strip():
print("ERROR: No reference text provided", file=sys.stderr)
sys.exit(1)
# Parse to BibTeX
if args.llm:
bibtex = send_to_llm(ref_text, pdf_path=args.pdf or "")
count = 0
else:
bibtex, count = parse_to_bibtex(ref_text, pdf_path=args.pdf or "")
# Write output
outfile = open(args.output, "w") if args.output else sys.stdout
try:
if not args.llm and not args.output:
print(f"# {count} references extracted", file=sys.stderr)
outfile.write(bibtex + "\n" if bibtex else "")
finally:
if args.output:
outfile.close()
if __name__ == "__main__":
main()