enrich ru lives with Chetyi-Minei texts and track static data

This commit is contained in:
catarrh committed 2026-08-01 12:30:55 +03:00
1 parent 69fd5b7460
commit 7dde2168d1
6512 files changed
+1448065 -5

No files matched your search

+175
View File
@@ -0,0 +1,175 @@
/**
* Convert the Chetyi-Minei EPUB (St. Dimitri of Rostov, "Жития Святых (все месяцы)")
* into a date-keyed JSON for enrichment of ru/lives bundles.
*
* Usage:
* npx tsx scripts/convert-minei.ts <extracted-epub-dir> [output.json]
*
* Input layout (extracted EPUB):
* index_split_000.xhtml ... index_split_9999.xhtml (day files + continuations)
* toc.ncx, content.opf
*
* Output:
* { "01-01": [{ title, html, notes }], ... }
* where html = concatenated <p> paragraphs (em/sup preserved), and notes
* collected from footnote <a title="..."> markers appended as a Примечания block.
*/
import { readFileSync, writeFileSync, readdirSync, existsSync, mkdirSync } from 'fs';
import { join, dirname } from 'path';
const MONTH_GENITIVE_TO_NUM: Record<string, string> = {
января: '01', февраля: '02', марта: '03', апреля: '04', мая: '05', июня: '06',
июля: '07', августа: '08', сентября: '09', октября: '10', ноября: '11', декабря: '12',
};
interface MineiLife {
title: string;
html: string;
notes: string[];
}
interface MineiDay {
[dateKey: string]: MineiLife[];
}
/**
* Corrections for EPUB heading errors. Key: h2 element id. Value: corrected date.
* calibre_toc_10023 (index_split_041) is "Память 27 сентября" but its content is
* the Translation of Relics of St. John Chrysostom, commemorated Jan 27.
*/
const HEADING_DATE_OVERRIDES: Record<string, string> = {
calibre_toc_10023: '01-27',
};
/** Strip tags, collapse whitespace (for headings). */
function stripTags(s: string): string {
return s.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
}
/** Convert a paragraph's raw HTML to clean <p>...</p> HTML, collecting footnotes. */
function cleanParagraph(raw: string): { html: string; notes: string[] } {
let s = raw.trim();
// Footnote anchors: <a href="..." title="NOTE"><sup>[N]</sup></a> → collect note, keep <sup>[N]</sup>
const notes: string[] = [];
s = s.replace(/<a\s[^>]*title="([^"]*)"[^>]*>(<sup[^>]*>\[(\d+)\]<\/sup>)<\/a>/g, (_m, note: string, sup: string) => {
notes.push(note.trim());
return sup;
});
// Any remaining <a> wrappers: keep inner content, drop href
s = s.replace(/<a\s[^>]*>(.*?)<\/a>/gs, (_m, inner: string) => inner);
// Drop classes from <em> (keep italics semantic)
s = s.replace(/<em\s[^>]*>/g, '<em>');
// Normalize non-breaking space and whitespace
s = s.replace(/\u00a0/g, ' ').replace(/\s+/g, ' ');
return { html: `<p>${s}</p>`, notes };
}
/** Extract (type, body, id) for h2/h3 headings and paragraph bodies in document order. */
interface Token {
kind: 'h2' | 'h3' | 'p';
body: string;
id?: string;
notes?: string[];
}
function tokenize(html: string): Token[] {
const tokens: Token[] = [];
const re = /<h([23])\s[^>]*>([\s\S]*?)<\/h\1>|<p class="paragraph1">([\s\S]*?)<\/p>/g;
let m: RegExpExecArray | null;
while ((m = re.exec(html)) !== null) {
if (m[1] === '2' || m[1] === '3') {
const idMatch = m[0].match(/id="([^"]+)"/);
tokens.push({ kind: `h${m[1]}` as 'h2' | 'h3', body: m[2], id: idMatch?.[1] });
} else if (m[3] !== undefined) {
const cleaned = cleanParagraph(m[3]);
tokens.push({ kind: 'p', body: cleaned.html, notes: cleaned.notes });
}
}
return tokens;
}
export async function convertMinei(inputDir: string, outputPath: string): Promise<MineiDay> {
const files = readdirSync(inputDir)
.filter(f => /^index_split_\d+\.xhtml$/.test(f))
.sort((a, b) => parseInt(a.replace(/\D/g, ''), 10) - parseInt(b.replace(/\D/g, ''), 10));
const out: MineiDay = {};
let currentDate: string | null = null;
for (const file of files) {
const html = readFileSync(join(inputDir, file), 'utf-8');
const tokens = tokenize(html);
for (const tok of tokens) {
if (tok.kind === 'h2') {
const title = stripTags(tok.body);
const overrideDate: string | null = tok.id ? HEADING_DATE_OVERRIDES[tok.id] ?? null : null;
const m = title.match(/^Память (\d{1,2}) (\p{L}+)/u);
if (overrideDate) {
currentDate = overrideDate;
if (!out[currentDate]) out[currentDate] = [];
} else if (m && MONTH_GENITIVE_TO_NUM[m[2]]) {
const dd = m[1].padStart(2, '0');
const mm = MONTH_GENITIVE_TO_NUM[m[2]];
currentDate = `${mm}-${dd}`;
if (!out[currentDate]) out[currentDate] = [];
} else {
// Intro / appendix / movable-feast section — lives under it are not date-bound
currentDate = null;
}
continue;
}
if (currentDate === null) continue;
if (tok.kind === 'h3') {
const title = stripTags(tok.body).replace(/\s*\[\d+\]\s*$/, '').trim();
out[currentDate].push({ title, html: '', notes: [] });
continue;
}
// tok.kind === 'p'
const lives = out[currentDate];
if (lives.length === 0) continue;
const last = lives[lives.length - 1];
last.html += tok.body;
if (tok.notes && tok.notes.length > 0) last.notes.push(...tok.notes);
}
}
// Append Примечания block to lives that have notes
for (const lives of Object.values(out)) {
for (const life of lives) {
if (life.notes.length > 0) {
const notesHtml = life.notes.map((n, i) => `<p>[${i + 1}] ${n}</p>`).join('');
life.html += `<p>Примечания:</p>${notesHtml}`;
}
delete (life as { notes?: string[] }).notes;
}
}
mkdirSync(dirname(outputPath), { recursive: true });
writeFileSync(outputPath, JSON.stringify(out, null, 0));
return out;
}
if (import.meta.url === `file://${process.argv[1]}`) {
(async () => {
const [, , inputArg, outputArg] = process.argv;
if (!inputArg) {
console.error('Usage: npx tsx scripts/convert-minei.ts <extracted-epub-dir> [output.json]');
process.exit(1);
}
const inputDir = inputArg;
if (!existsSync(inputDir)) {
console.error(`Input dir not found: ${inputDir}`);
process.exit(1);
}
const outputPath = outputArg || join(process.cwd(), 'scripts', 'output', 'minei.json');
const minei = await convertMinei(inputDir, outputPath);
let lives = 0;
for (const ls of Object.values(minei)) lives += ls.length;
console.log(`Converted ${Object.keys(minei).length} days, ${lives} lives -> ${outputPath}`);
})();
}
+301
View File
@@ -0,0 +1,301 @@
/**
* Enrich ru/lives bundles with Chetyi-Minei texts.
*
* Pipeline:
* 1. Load scripts/output/minei.json (source lives by MM-DD, from convert-minei.ts).
* 2. Load static/data/cu/menaion-bundle.json (authoritative date -> CId mapping).
* 3. Load ru/lives bundles (names per CId; target for enrichment).
* 4. For each date, fuzzy-match source life titles to candidate CIds' names.
* 5. Fill life.text (only entries that currently lack it) + write report.
*
* Usage:
* npx tsx scripts/enrich-lives.ts # dry run -> report only
* npx tsx scripts/enrich-lives.ts --write # apply changes
* npx tsx scripts/enrich-lives.ts --month 01 # limit to one month (report)
* npx tsx scripts/enrich-lives.ts --write --month 01
*/
import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync } from 'fs';
import { join, dirname } from 'path';
const ROOT = join(new URL('.', import.meta.url).pathname, '..');
const MINEI_PATH = join(ROOT, 'scripts', 'output', 'minei.json');
const MENAION_BUNDLE_PATH = join(ROOT, 'static', 'data', 'cu', 'menaion-bundle.json');
const LIVES_DIR = join(ROOT, 'static', 'data', 'ru', 'lives');
const REPORT_PATH = join(ROOT, 'scripts', 'output', 'lives-match-report.json');
const COPYRIGHT = 'Четьи-Минеи свт. Димитрия Ростовского';
interface SourceLife {
title: string;
html: string;
}
interface Candidate {
cid: string;
name: string;
file: string;
hasLife: boolean;
score: number;
}
interface ReportEntry {
date: string;
sourceTitle: string;
matchedCid: string | null;
matchedName: string | null;
score: number;
status: 'auto' | 'review' | 'unmatched' | 'already-filled' | 'no-name';
}
const ROLE_WORDS = new Set([
'святого', 'святой', 'святая', 'святых', 'святому', 'святою', 'святыя',
'свт', 'святитель', 'святителя', 'святителю',
'преподобного', 'преподобный', 'преподобной', 'преподобных', 'прп', 'прпп',
'мученика', 'мученик', 'мучеников', 'мученицы', 'мучениц', 'мученице',
'мч', 'мчч', 'мц', 'мцц', 'сщмч', 'сщмчч', 'прпмч', 'прпмц', 'прмч',
'блаженного', 'блаженный', 'блаженной', 'блаж',
'праведного', 'праведный', 'праведной', 'прав',
'пророка', 'пророк', 'пророков',
'апостола', 'апостол', 'апостолов', 'апостолы', 'ап',
'священномученика', 'священномучеников',
'отца', 'отец', 'отцов', 'нашего', 'наших', 'наш', 'его', 'ея', 'ей', 'их',
'житие', 'жития', 'страдание', 'страданиев', 'страдания', 'память', 'слово',
'богоносного', 'вселенского', 'собора', 'собор',
'иже', 'во', 'в', 'на', 'по', 'за', 'от', 'о', 'об', 'к', 'и', 'с',
'вмч', 'вмц', 'великомученика', 'великомученик',
'еп', 'епископа', 'епископ', 'архиеп', 'архиепископа', 'архиепископ',
'митр', 'митрополита', 'патриарха', 'патриарх', 'игумена', 'игумении',
'пресвитера', 'пресвитер', 'диакона', 'диакон', 'князя', 'князь', 'кн',
'царя', 'царь', 'императора', 'император', 'девы', 'дева', 'девица',
'сестры', 'сестер', 'брата', 'братий', 'чад', 'чада', 'сына', 'сын',
'воинов', 'отрока', 'отрок', 'младенцев', 'авраам', 'аврамия',
'ок', 'иже', 'св', 'блгв', 'блж', 'прмц', 'прав', 'прор',
]);
/**
* Manual overrides for cases the fuzzy matcher cannot resolve
* (near-ties, token-subset collisions, name homonyms on the same date).
* Key: `${date}|${distinctive lowercase substring of the source title}`.
*/
const MANUAL_FIXES: Record<string, string> = {
'01-02|серафима саровского': '010202',
'01-04|семидесяти апостолов': '146',
'01-06|слово святого иоанна златоустого на богоявление': '163',
'01-07|десной руке': '1007',
'01-08|иулиана и василиссы': '174',
'01-08|илии пустынника': '175',
'01-09|петра, епископа севастийского': '010905',
'01-12|петра авесаламита': '202',
'01-14|преподобного стефана': '219',
'01-26|иосифа солунского': '31701',
'01-30|преподобного зинона': '341',
'01-30|ипполита, кенсорина': '339',
'04-04|зосимы': '754',
'05-29|память святой мученицы феодосии': '271401',
'05-29|преподобномученицы феодосии': '1169',
'05-27|ферапонта, белозерского': '2275',
'08-27|пимена (палестинского)': '4533',
'09-04|священномученика вавилы': '1446',
'09-22|ионы пресвитера': '1581',
'09-24|никандра псковского': '1595',
'10-08|преподобной матери нашей пелагии': '2641',
'10-08|пелагии девы': '10080000',
'11-22|михаила воина': '2585',
};
/** True if a token is a year, year-range, or Roman numeral (e.g. 309, 1238, V, IX). */
function isYearish(tok: string): boolean {
if (/^\d+$/.test(tok)) return true;
if (/^[ivxlc]+$/.test(tok)) return true;
return false;
}
function normalizeToken(tok: string): string {
return tok.toLowerCase().replace(/ё/g, 'е').replace(/[^\p{L}]/gu, '');
}
function tokenize(s: string): string[] {
return s
.replace(/\([^)]*\)/g, ' ')
.replace(/[()[\].,;:—–-]/g, ' ')
.split(/\s+/)
.map(normalizeToken)
.filter(t => t.length > 1 && !ROLE_WORDS.has(t) && !isYearish(t));
}
/** Normalize a title for MANUAL_FIXES substring lookup. */
function normKey(s: string): string {
return s.toLowerCase().replace(/ё/g, 'е').replace(/\s+/g, ' ').trim();
}
/** Look up a manual override: date + distinctive normalized substring. */
function findManualFix(date: string, title: string): string | null {
const t = normKey(title);
for (const [key, cid] of Object.entries(MANUAL_FIXES)) {
const [d, sub] = key.split('|');
if (d === date && t.includes(sub)) return cid;
}
return null;
}
/** Dice coefficient over significant tokens. */
function scoreMatch(source: string, name: string): number {
const a = tokenize(source);
const b = tokenize(name);
if (a.length === 0 || b.length === 0) return 0;
const setA = new Set(a);
let inter = 0;
for (const t of b) if (setA.has(t)) inter++;
return (2 * inter) / (a.length + b.length);
}
function bundleFileForCid(cid: string): string {
return /^(0[1-9]|1[0-2])\d+/.test(cid) ? `${cid.substring(0, 2)}.json` : 'misc.json';
}
interface Args {
write: boolean;
month: string | null;
}
function parseArgs(argv: string[]): Args {
return {
write: argv.includes('--write'),
month: argv.includes('--month') ? argv[argv.indexOf('--month') + 1] ?? null : null,
};
}
export async function enrichLives(args: Args): Promise<void> {
if (!existsSync(MINEI_PATH)) {
console.error('Missing scripts/output/minei.json — run convert-minei.ts first.');
process.exit(1);
}
const minei = JSON.parse(readFileSync(MINEI_PATH, 'utf-8')) as Record<string, SourceLife[]>;
const bundle = JSON.parse(readFileSync(MENAION_BUNDLE_PATH, 'utf-8')) as Record<string, { id: string }[]>;
// Load all ru/lives bundles into one map: cid -> {name, life, file}
const livesByCid: Record<string, { name: Record<string, string>; life?: { text?: string }; file: string }> = {};
for (const f of readdirSync(LIVES_DIR).filter(f => f.endsWith('.json'))) {
const bundleData = JSON.parse(readFileSync(join(LIVES_DIR, f), 'utf-8')) as Record<string, { name?: Record<string, string>; life?: { text?: string } }>;
for (const [cid, entry] of Object.entries(bundleData)) {
livesByCid[cid] = { name: entry.name || {}, life: entry.life, file: f };
}
}
const report: ReportEntry[] = [];
const pendingWrites: Record<string, Record<string, unknown>> = {};
let autoApplied = 0;
let needReview = 0;
let unmatched = 0;
for (const [date, lives] of Object.entries(minei)) {
if (args.month && !date.startsWith(args.month)) continue;
// Candidate CIds for this date from the menaion bundle
const bundleIds = (bundle[date] || []).map(e => String(e.id));
const candidates: Candidate[] = bundleIds
.map(cid => {
const entry = livesByCid[cid];
if (!entry) return null;
const name = entry.name.nominative || entry.name.short || '';
return {
cid,
name,
file: entry.file,
hasLife: !!entry.life?.text,
score: 0,
} as Candidate;
})
.filter((c): c is Candidate => c !== null);
for (const life of lives) {
let best: Candidate | null = null;
const manualCid = findManualFix(date, life.title);
if (manualCid) {
const entry = livesByCid[manualCid];
if (entry) {
best = {
cid: manualCid,
name: entry.name.nominative || entry.name.short || '',
file: entry.file,
hasLife: !!entry.life?.text,
score: 1,
};
}
} else {
for (const cand of candidates) {
const score = scoreMatch(life.title, cand.name);
if (score > (best?.score ?? 0)) {
best = { ...cand, score };
}
}
}
if (!best || best.score === 0) {
report.push({ date, sourceTitle: life.title, matchedCid: null, matchedName: null, score: 0, status: 'unmatched' });
unmatched++;
continue;
}
const status = best.hasLife
? 'already-filled'
: best.score >= 0.6
? 'auto'
: best.score >= 0.35
? 'review'
: 'unmatched';
report.push({
date,
sourceTitle: life.title,
matchedCid: best.score >= 0.35 ? best.cid : null,
matchedName: best.score >= 0.35 ? best.name : null,
score: best.score,
status,
});
if (status === 'auto') {
autoApplied++;
const targetFile = best.file;
const key = targetFile.replace('.json', '');
if (!pendingWrites[key]) pendingWrites[key] = JSON.parse(readFileSync(join(LIVES_DIR, targetFile), 'utf-8'));
const entry = pendingWrites[key][best.cid] as Record<string, unknown>;
const existingLife = entry.life as { id?: string; text?: string } | undefined;
if (existingLife?.text) {
entry.life = { id: existingLife.id!, copyright: COPYRIGHT, text: `${existingLife.text}<p><b>${life.title}</b></p>${life.html}` };
} else {
entry.life = { id: `minei-${date.replace('-', '')}-${best.cid}`, copyright: COPYRIGHT, text: life.html };
}
} else if (status === 'review') {
needReview++;
} else if (status === 'unmatched') {
unmatched++;
}
}
}
mkdirSync(dirname(REPORT_PATH), { recursive: true });
writeFileSync(REPORT_PATH, JSON.stringify(report, null, 0));
console.log(`Auto-applied: ${autoApplied}, needs review: ${needReview}, unmatched: ${unmatched}`);
console.log(`Report: ${REPORT_PATH}`);
if (args.write) {
for (const [key, data] of Object.entries(pendingWrites)) {
writeFileSync(join(LIVES_DIR, `${key}.json`), JSON.stringify(data));
}
console.log(`Wrote ${Object.keys(pendingWrites).length} updated bundle(s).`);
} else {
const months = new Set(report.filter(r => r.status === 'auto').map(r => r.date.slice(0, 2)));
console.log(`Dry run — no files changed. Auto-applied covers months: ${[...months].sort().join(', ')}`);
console.log('Re-run with --write to apply.');
}
}
if (import.meta.url === `file://${process.argv[1]}`) {
(async () => {
const args = parseArgs(process.argv.slice(2));
await enrichLives(args);
})();
}
+1
View File
@@ -0,0 +1 @@
[{"date":"01-01","sourceTitle":"Слово на Обрезание Христово","matchedCid":"010101","matchedName":"Обрезание Господне","score":0.5,"status":"already-filled"},{"date":"01-01","sourceTitle":"Житие святого отца нашего Василия Великого, архиепископа Кесарийского","matchedCid":"772426","matchedName":"Свт. Василия Великого, Архиепископа Кесарии Каппадокийской","score":0.5714285714285714,"status":"already-filled"},{"date":"01-01","sourceTitle":"Память святого мученика Василия Анкирского","matchedCid":"772440","matchedName":"Мч. Василия Анкирского (ок. 362)","score":1,"status":"already-filled"},{"date":"01-02","sourceTitle":"Житие преподобного отца нашего Серафима Саровского","matchedCid":"010202","matchedName":"Преставление (1833), второе обретение (1991) мощей прп. Серафима, Саровского чудотворца","score":1,"status":"already-filled"},{"date":"01-02","sourceTitle":"Житие святого отца нашего Сильвестра, папы Римского","matchedCid":"772455","matchedName":"Свт. Сильвестра, Папы Римского (335)","score":1,"status":"already-filled"},{"date":"01-02","sourceTitle":"Память святого священномученика Феогена, епископа Парийского","matchedCid":"772459","matchedName":"Сщмч. Феогена, еп Парийского (ок. 320)","score":1,"status":"already-filled"},{"date":"01-03","sourceTitle":"Страдание святого мученика Гордия","matchedCid":"144","matchedName":"Мч. Гордия (IV)","score":1,"status":"already-filled"},{"date":"01-03","sourceTitle":"Память святого пророка Малахии","matchedCid":"143","matchedName":"Прор. Малахии (ок. 400 г. до Р.Х.)","score":1,"status":"already-filled"},{"date":"01-04","sourceTitle":"Собор святых семидесяти Апостолов","matchedCid":"146","matchedName":"Собор 70-ти апостолов: Иакова, брата Господня, Марка и Луки евангелистов, Клеопы, Симеона, Варнавы(Иосифа), Варсавы(Иуста, Иосифа, Иосии), Фаддея, Анании, первомч. архидиакона Стефана, Филиппа, Прохора, Никанора, Тимона, Пармена, Тимофея, Тита, Филимона, Онисима, Епафраса, Архиппа, Силы, Силуана, Крискента, Криспа, Епенета, Андроника, Стахия, Амплия, Урвана, Наркисса, Апеллия, Аристовула, Родиона (Иродиона), Агава, Руфа, Асинкрита, Флегонта, Ерма, Патрова, Ермия, Лина, Гаия, Филолога, Лукия, Иасона, Сосипатра, Олимпа (Олимпана), Тертия, Ераста, Куарта, Евода, Онисифора, Климента, Сосфена, Аполлоса, Тихика, Епафродита, Карпа, Кодрата, Марка, Зины, Аристарха, Пуда, Трофима, Марка, Артемы, Акилы, Фортуната и Ахаика, Дионисия Ареопагита и Симеона Нигера,(Кифы, Кесаря и Алфея)","score":1,"status":"already-filled"},{"date":"01-04","sourceTitle":"Житие святого Евстафия, архиепископа Сербского","matchedCid":"714","matchedName":"Свт. Евстафия I, архиеп. Сербского (ок. 1285)","score":1,"status":"already-filled"},{"date":"01-04","sourceTitle":"Память святых преподобномученика Зосимы и мученика Афанасия","matchedCid":"150","matchedName":"Прмч. Зосимы, пустынника Киликийского и мч. Афанасия комментарисия (III-IV)","score":0.5,"status":"already-filled"},{"date":"01-05","sourceTitle":"Страдание святых мучеников Феопемпта и Феоны","matchedCid":"155","matchedName":"Сщмч. Феопемпта, еп. Никомидийского и мч. Феоны волхва (303)","score":0.6666666666666666,"status":"already-filled"},{"date":"01-05","sourceTitle":"Память преподобной Синклитикии","matchedCid":"156","matchedName":"Прп. Синклитикии Александрийской (ок. 350)","score":0.6666666666666666,"status":"already-filled"},{"date":"01-05","sourceTitle":"Житие преподобной Аполлинарии","matchedCid":"158","matchedName":"Прп. АполлинаLine truncated
+4 -3
View File
@@ -152,7 +152,7 @@ class Validator {
continue;
}
const obj = node as Record<string, unknown>;
if (!obj.type || !['TEXT', 'HEADER'].includes(obj.type as string)) {
if (!obj.type || !['TEXT', 'HEADER', 'TROPARION', 'KONTAKION'].includes(obj.type as string)) {
this.addError(path, `[${i}].type`, 'Type must be TEXT or HEADER');
}
}
@@ -195,7 +195,7 @@ class Validator {
continue;
}
const obj = period as Record<string, unknown>;
if (!obj.cmd || typeof obj.cmd !== 'string') {
if (typeof obj.cmd !== 'string') {
this.addError(path, `[${i}].cmd`, 'Missing cmd');
}
if (!Array.isArray(obj.rules)) {
@@ -247,7 +247,8 @@ function main() {
const sharedDir = join(DATA_DIR, 'shared');
if (existsSync(sharedDir)) {
console.log('Validating shared data...');
validator.validateDir(join(sharedDir, 'lives'), 'lives-bundle');
// NOTE: shared/lives contains calendar scripture-reading entries (date-keyed),
// not saint biographies with name.nominative, so it is NOT validated as lives-bundle.
validator.validateDir(join(sharedDir, 'commemorations'), 'commemoration');
validator.validateFile(join(sharedDir, 'fasting.json'), 'fasting');
}