forked/topola-viewer
1
0
Fork 0
mirror of https://github.com/PeWu/topola-viewer.git synced 2026-05-26 23:26:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor wikitree.ts into smaller files

Browse Source 
#vibecoded
This commit is contained in:
Przemek Więch
2026-04-28 17:43:14 +02:00
parent f3b2b436b6
commit 5a1695fbc1
4 changed files with 868 additions and 804 deletions
Download Patch File Download Diff File Expand all files Collapse all files

286
src/datasource/gedcom_generator.ts Normal file
View File

@@ -0,0 +1,286 @@
import {GedcomEntry} from 'parse-gedcom';
import {
Date,
DateOrRange,
JsonEvent,
JsonFam,
JsonGedcomData,
JsonImage,
JsonIndi,
} from 'topola';
import {isValidDateOrRange} from '../util/date_util';
import {GedcomData} from '../util/gedcom_util';
const MONTHS = new Map<number, string>([
[1, 'JAN'],
[2, 'FEB'],
[3, 'MAR'],
[4, 'APR'],
[5, 'MAY'],
[6, 'JUN'],
[7, 'JUL'],
[8, 'AUG'],
[9, 'SEP'],
[10, 'OCT'],
[11, 'NOV'],
[12, 'DEC'],
]);
function dateToGedcom(date: Date): string {
return [date.qualifier, date.day, MONTHS.get(date.month!), date.year]
.filter((x) => x !== undefined)
.join(' ');
}
function dateOrRangeToGedcom(dateOrRange: DateOrRange): string {
if (dateOrRange.date) {
return dateToGedcom(dateOrRange.date);
}
if (!dateOrRange.dateRange) {
return '';
}
if (dateOrRange.dateRange.from && dateOrRange.dateRange.to) {
return `BET ${dateToGedcom(dateOrRange.dateRange.from)} AND ${
dateOrRange.dateRange.to
}`;
}
if (dateOrRange.dateRange.from) {
return `AFT ${dateToGedcom(dateOrRange.dateRange.from)}`;
}
if (dateOrRange.dateRange.to) {
return `BEF ${dateToGedcom(dateOrRange.dateRange.to)}`;
}
return '';
}
function nameToGedcom(type: string, firstName?: string, lastName?: string) {
return {
level: 1,
pointer: '',
tag: 'NAME',
data: `${firstName || ''} /${lastName || ''}/`,
tree: [
{
level: 2,
pointer: '',
tag: 'TYPE',
data: type,
tree: [],
},
],
};
}
function eventToGedcom(event: JsonEvent): GedcomEntry[] {
const result = [];
if (isValidDateOrRange(event)) {
result.push({
level: 2,
pointer: '',
tag: 'DATE',
data: dateOrRangeToGedcom(event),
tree: [],
});
}
if (event.place) {
result.push({
level: 2,
pointer: '',
tag: 'PLAC',
data: event.place,
tree: [],
});
}
return result;
}
function imageToGedcom(
image: JsonImage,
fullSizePhotoUrl: string | undefined,
): GedcomEntry[] {
return [
{
level: 2,
pointer: '',
tag: 'FILE',
data: fullSizePhotoUrl || image.url,
tree: [
{
level: 3,
pointer: '',
tag: 'FORM',
data: image.title?.split('.').pop() || '',
tree: [],
},
{
level: 3,
pointer: '',
tag: 'TITL',
data: image.title?.split('.')[0] || '',
tree: [],
},
],
},
];
}
function indiToGedcom(
indi: JsonIndi,
fullSizePhotoUrl: Map<string, string>,
personNames: {birth?: string; married?: string; aka?: string},
): GedcomEntry {
// WikiTree URLs replace spaces with underscores.
const escapedId = indi.id.replace(/ /g, '_');
const record: GedcomEntry = {
level: 0,
pointer: `@${indi.id}@`,
tag: 'INDI',
data: '',
tree: [],
};
if (personNames.birth) {
record.tree.push(nameToGedcom('birth', indi.firstName, personNames.birth));
}
if (personNames.married) {
record.tree.push(
nameToGedcom('married', indi.firstName, personNames.married),
);
}
if (personNames.aka) {
record.tree.push(nameToGedcom('aka', indi.firstName, personNames.aka));
}
if (indi.birth) {
record.tree.push({
level: 1,
pointer: '',
tag: 'BIRT',
data: '',
tree: eventToGedcom(indi.birth),
});
}
if (indi.death) {
record.tree.push({
level: 1,
pointer: '',
tag: 'DEAT',
data: '',
tree: eventToGedcom(indi.death),
});
}
if (indi.famc) {
record.tree.push({
level: 1,
pointer: '',
tag: 'FAMC',
data: `@${indi.famc}@`,
tree: [],
});
}
(indi.fams || []).forEach((fams) =>
record.tree.push({
level: 1,
pointer: '',
tag: 'FAMS',
data: `@${fams}@`,
tree: [],
}),
);
if (!indi.id.startsWith('~')) {
record.tree.push({
level: 1,
pointer: '',
tag: 'WWW',
data: `https://www.wikitree.com/wiki/${escapedId}`,
tree: [],
});
}
(indi.images || []).forEach((image) => {
record.tree.push({
level: 1,
pointer: '',
tag: 'OBJE',
data: '',
tree: imageToGedcom(image, fullSizePhotoUrl.get(indi.id)),
});
});
return record;
}
function famToGedcom(fam: JsonFam): GedcomEntry {
const record: GedcomEntry = {
level: 0,
pointer: `@${fam.id}@`,
tag: 'FAM',
data: '',
tree: [],
};
if (fam.wife) {
record.tree.push({
level: 1,
pointer: '',
tag: 'WIFE',
data: `@${fam.wife}@`,
tree: [],
});
}
if (fam.husb) {
record.tree.push({
level: 1,
pointer: '',
tag: 'HUSB',
data: `@${fam.husb}@`,
tree: [],
});
}
(fam.children || []).forEach((child) =>
record.tree.push({
level: 1,
pointer: child,
tag: 'CHILD',
data: '',
tree: [],
}),
);
if (fam.marriage) {
record.tree.push({
level: 1,
pointer: '',
tag: 'MARR',
data: '',
tree: eventToGedcom(fam.marriage),
});
}
return record;
}
/**
* Creates a GEDCOM structure for the purpose of displaying the details
* panel.
*/
export function buildGedcom(
data: JsonGedcomData,
fullSizePhotoUrls: Map<string, string>,
personNames: Map<string, {birth?: string; married?: string; aka?: string}>,
): GedcomData {
const gedcomIndis: {[key: string]: GedcomEntry} = {};
const gedcomFams: {[key: string]: GedcomEntry} = {};
data.indis.forEach((indi) => {
gedcomIndis[indi.id] = indiToGedcom(
indi,
fullSizePhotoUrls,
personNames.get(indi.id) || {},
);
});
data.fams.forEach((fam) => {
gedcomFams[fam.id] = famToGedcom(fam);
});
return {
head: {level: 0, pointer: '', tag: 'HEAD', data: '', tree: []},
indis: gedcomIndis,
fams: gedcomFams,
other: {},
};
}

815
src/datasource/wikitree.ts
View File

@@ -1,372 +1,22 @@
import {GedcomEntry} from 'parse-gedcom';
import {IntlShape} from 'react-intl'; import {IntlShape} from 'react-intl';
import {
Date,
DateOrRange,
JsonEvent,
JsonFam,
JsonGedcomData,
JsonImage,
JsonIndi,
} from 'topola';
import {StringUtils} from 'turbocommons-ts';
import {
clientLogin,
getLoggedInUserName,
getPeople,
getRelatives as getRelativesApi,
Person,
} from 'wikitree-js';
import {analyticsEvent} from '../util/analytics'; import {analyticsEvent} from '../util/analytics';
import {isValidDateOrRange} from '../util/date_util';
import {TopolaError} from '../util/error'; import {TopolaError} from '../util/error';
import {GedcomData, normalizeGedcom, TopolaData} from '../util/gedcom_util'; import {normalizeGedcom, TopolaData} from '../util/gedcom_util';
import {DataSource, DataSourceEnum, SourceSelection} from './data_source'; import {DataSource, DataSourceEnum, SourceSelection} from './data_source';
import {loadData, PRIVATE_ID_PREFIX} from './wikitree_api';
import {convertFams, convertIndis, convertPersonNames} from './wikitree_transformer';
import {buildGedcom} from './gedcom_generator';
const WIKITREE_APP_ID = 'topola-viewer'; export {PRIVATE_ID_PREFIX};
/** Prefix for IDs of private individuals. */
export const PRIVATE_ID_PREFIX = '~Private';
/** Gets item from session storage. Logs exception if one is thrown. */
function getSessionStorageItem(key: string): string | null {
try {
return sessionStorage.getItem(key);
} catch (e) {
console.warn('Failed to load data from session storage: ' + e);
}
return null;
}
/** Sets item in session storage. Logs exception if one is thrown. */
function setSessionStorageItem(key: string, value: string) {
try {
sessionStorage.setItem(key, value);
} catch (e) {
console.warn('Failed to store data in session storage: ' + e);
}
}
function getApiOptions(handleCors: boolean) {
return Object.assign(
{appId: WIKITREE_APP_ID},
handleCors
? {
apiUrl:
'https://topolaproxy.bieda.it/https://api.wikitree.com/api.php',
}
: {},
);
}
/** /**
* Retrieves ancestors from WikiTree for the given person ID. * Main entrypoint for loading data from WikiTree and transforming it to Topola format.
* Uses sessionStorage for caching responses. *
* @param key The person key to load.
* @param intl Intl shape for localization.
* @param authcode Optional authentication code.
* @returns Promise resolving to transformed Topola data.
*/ */
async function getAncestors(
key: string,
handleCors: boolean,
): Promise<Person[]> {
// Limit the number of generations of ancestors.
const ancestorsGenerationLimit = 5;
const cacheKey = `wikitree:ancestors:${key}`;
const cachedData = getSessionStorageItem(cacheKey);
if (cachedData) {
return JSON.parse(cachedData);
}
const result = await getPeople(
[key],
{ancestors: ancestorsGenerationLimit},
getApiOptions(handleCors),
);
setSessionStorageItem(cacheKey, JSON.stringify(result));
return result;
}
/**
* Retrieves relatives from WikiTree for the given array of person IDs.
* Uses sessionStorage for caching responses.
*/
async function getRelatives(
keys: string[],
handleCors: boolean,
): Promise<Person[]> {
const result: Person[] = [];
const keysToFetch: string[] = [];
keys.forEach((key) => {
const cachedData = getSessionStorageItem(`wikitree:relatives:${key}`);
if (cachedData) {
result.push(JSON.parse(cachedData));
} else {
keysToFetch.push(key);
}
});
if (keysToFetch.length === 0) {
return result;
}
const response = await getRelativesApi(
keysToFetch,
{getChildren: true, getSpouses: true},
getApiOptions(handleCors),
);
if (!response) {
const id = keysToFetch[0];
throw new TopolaError(
'WIKITREE_PROFILE_NOT_FOUND',
`WikiTree profile ${id} not found`,
{id},
);
}
response.forEach((person) => {
setSessionStorageItem(
`wikitree:relatives:${person.Name}`,
JSON.stringify(person),
);
});
return result.concat(response);
}
async function logInIfNeeded(
authcode: string | undefined,
handleCors: boolean,
) {
if (!handleCors && !getLoggedInUserName() && authcode) {
const loginResult = await clientLogin(authcode, {appId: WIKITREE_APP_ID});
if (loginResult.result === 'Success') {
sessionStorage.clear();
}
}
}
async function getFirstPerson(key: string, handleCors: boolean) {
const person = (await getRelatives([key], handleCors))[0];
if (!person?.Name) {
const id = key;
throw new TopolaError(
'WIKITREE_PROFILE_NOT_ACCESSIBLE',
`WikiTree profile ${id} is not accessible. Try logging in.`,
{id},
);
}
return person;
}
function getSpouseKeys(person: Person) {
return Object.values(person.Spouses || {}).map((s) => s.Name);
}
async function getAllAncestors(keys: string[], handleCors: boolean) {
const ancestors = await Promise.all(
keys.map((key) => getAncestors(key, handleCors)),
);
const ancestorKeys = ancestors
.flat()
.map((person) => person.Name)
.filter((key) => !!key);
const ancestorDetails = await getRelatives(ancestorKeys, handleCors);
// Map from person id to father id if the father profile is private.
const privateFathers: Map<number, number> = new Map();
// Map from person id to mother id if the mother profile is private.
const privateMothers: Map<number, number> = new Map();
// Andujst private individual ids so that there are no collisions in the case
// that ancestors were collected for more than one person.
ancestors.forEach((ancestorList, index) => {
const offset = 1000 * index;
// Adjust ids by offset.
ancestorList.forEach((person) => {
if (person.Id < 0) {
person.Id -= offset;
person.Name = `${PRIVATE_ID_PREFIX}${person.Id}`;
}
if (person.Father < 0) {
person.Father -= offset;
privateFathers.set(person.Id, person.Father);
}
if (person.Mother < 0) {
person.Mother -= offset;
privateMothers.set(person.Id, person.Mother);
}
});
});
// Set the Father and Mother fields again because getRelatives doesn't return
// private parents.
ancestorDetails.forEach((person) => {
const privateFather = privateFathers.get(person.Id);
if (privateFather) {
person.Father = privateFather;
}
const privateMother = privateMothers.get(person.Id);
if (privateMother) {
person.Mother = privateMother;
}
});
// Collect private individuals.
const privateAncestors = ancestors.flat().filter((person) => person.Id < 0);
return ancestorDetails.concat(privateAncestors);
}
async function getAllDescendants(key: string, handleCors: boolean) {
const everyone: Person[] = [];
// Limit the number of generations of descendants because there may be tens of
// generations for some profiles.
const descendantGenerationLimit = 5;
// Fetch descendants recursively.
let toFetch = [key];
let generation = 0;
while (toFetch.length > 0 && generation <= descendantGenerationLimit) {
const people = await getRelatives(toFetch, handleCors);
everyone.push(...people);
const allSpouses = people.flatMap((person) =>
Object.values(person.Spouses || {}),
);
everyone.push(...allSpouses);
// Fetch all children.
toFetch = people.flatMap((person) =>
Object.values(person.Children || {}).map((c) => c.Name),
);
generation++;
}
return everyone;
}
async function loadData(key: string, authcode?: string) {
// Work around CORS if not in apps.wikitree.com domain.
const handleCors = window.location.hostname !== 'apps.wikitree.com';
await logInIfNeeded(authcode, handleCors);
const firstPerson = await getFirstPerson(key, handleCors);
const spouseKeys = getSpouseKeys(firstPerson);
// Fetch the ancestors of the input person and ancestors of his/her spouses.
const allAncestors = getAllAncestors([key].concat(spouseKeys), handleCors);
// Fetch descendants and their spouses.
const allDescendants = getAllDescendants(key, handleCors);
const everyone: Person[] = [
...(await allAncestors),
...(await allDescendants),
];
// Make sure the list contains unique elements.
return Array.from(
new Map(everyone.map((person) => [person.Id, person])).values(),
);
}
function getFamilies(people: Person[]) {
// Map from person id to the set of families where they are a spouse.
const families = new Map<number, Set<string>>();
people.forEach((person) => {
if (person.Mother || person.Father) {
const famId = getFamilyId(person.Mother, person.Father);
getSet(families, person.Mother).add(famId);
getSet(families, person.Father).add(famId);
}
if (person.Spouses) {
Object.values(person.Spouses).forEach((spouse) => {
const famId = getFamilyId(person.Id, spouse.Id);
getSet(families, person.Id).add(famId);
getSet(families, spouse.Id).add(famId);
});
}
});
return families;
}
function getChildren(people: Person[]) {
// Map from family id to the set of children.
const children = new Map<string, Set<number>>();
people.forEach((person) => {
if (person.Mother || person.Father) {
const famId = getFamilyId(person.Mother, person.Father);
getSet(children, famId).add(person.Id);
}
});
return children;
}
function getSpouses(people: Person[]) {
// Map from famliy id to the spouses.
const spouses = new Map<
string,
{wife?: number; husband?: number; spouse?: Person}
>();
people.forEach((person) => {
if (person.Mother || person.Father) {
const famId = getFamilyId(person.Mother, person.Father);
spouses.set(famId, {
wife: person.Mother || undefined,
husband: person.Father || undefined,
});
}
if (person.Spouses) {
Object.values(person.Spouses).forEach((spouse) => {
const famId = getFamilyId(person.Id, spouse.Id);
const familySpouses =
person.Gender === 'Male'
? {wife: spouse.Id, husband: person.Id, spouse}
: {wife: person.Id, husband: spouse.Id, spouse};
spouses.set(famId, familySpouses);
});
}
});
return spouses;
}
function convertIndis(people: Person[], intl: IntlShape) {
const families = getFamilies(people);
return people.map((person) => {
const indi = convertPerson(person, intl);
indi.fams = Array.from(getSet(families, person.Id));
return indi;
});
}
function convertFams(people: Person[]) {
// Map from numerical id to human-readable id.
const idToName = new Map(people.map((person) => [person.Id, person.Name]));
const children = getChildren(people);
const spouses = getSpouses(people);
return Array.from(spouses.entries()).map(([key, value]) => {
const fam: JsonFam = {
id: key,
};
const wife = value.wife && idToName.get(value.wife);
if (wife) {
fam.wife = wife;
}
const husband = value.husband && idToName.get(value.husband);
if (husband) {
fam.husb = husband;
}
fam.children = Array.from(getSet(children, key)).map(
(child) => idToName.get(child)!,
);
if (
value.spouse &&
((value.spouse.marriage_date &&
value.spouse.marriage_date !== '0000-00-00') ||
value.spouse.marriage_location)
) {
const parsedDate = parseDate(value.spouse.marriage_date);
fam.marriage = Object.assign({}, parsedDate, {
place: value.spouse.marriage_location,
});
}
return fam;
});
}
export async function loadWikiTree( export async function loadWikiTree(
key: string, key: string,
intl: IntlShape, intl: IntlShape,
@@ -396,450 +46,9 @@ export async function loadWikiTree(
return {chartData, gedcom}; return {chartData, gedcom};
} }
/** Creates a family identifier given 2 spouse identifiers. */
function getFamilyId(spouse1: number, spouse2: number) {
if (spouse2 > spouse1) {
return `${spouse1}_${spouse2}`;
}
return `${spouse2}_${spouse1}`;
}
function convertPerson(person: Person, intl: IntlShape): JsonIndi {
const indi: JsonIndi = {
id: person.Name,
};
if (person.Name.startsWith(PRIVATE_ID_PREFIX)) {
indi.hideId = true;
indi.firstName = intl.formatMessage({
id: 'wikitree.private',
defaultMessage: 'Private',
});
}
if (person.FirstName && person.FirstName !== 'Unknown') {
indi.firstName = person.FirstName;
} else if (person.RealName && person.RealName !== 'Unknown') {
indi.firstName = person.RealName;
}
if (person.LastNameAtBirth !== 'Unknown') {
indi.lastName = person.LastNameAtBirth;
}
if (person.Mother || person.Father) {
indi.famc = getFamilyId(person.Mother, person.Father);
}
if (person.Gender === 'Male') {
indi.sex = 'M';
} else if (person.Gender === 'Female') {
indi.sex = 'F';
}
if (
(person.BirthDate && person.BirthDate !== '0000-00-00') ||
person.BirthLocation ||
person.BirthDateDecade !== 'unknown'
) {
const parsedDate = parseDate(
person.BirthDate,
(person.DataStatus && person.DataStatus.BirthDate) || undefined,
);
const date = parsedDate || parseDecade(person.BirthDateDecade);
indi.birth = Object.assign({}, date, {place: person.BirthLocation});
}
if (
(person.DeathDate && person.DeathDate !== '0000-00-00') ||
person.DeathLocation ||
person.DeathDateDecade !== 'unknown'
) {
const parsedDate = parseDate(
person.DeathDate,
(person.DataStatus && person.DataStatus.DeathDate) || undefined,
);
const date = parsedDate || parseDecade(person.DeathDateDecade);
indi.death = Object.assign({}, date, {place: person.DeathLocation});
}
if (person.PhotoData) {
indi.images = [
{
url: `https://www.wikitree.com${person.PhotoData.url}`,
title: person.Photo,
},
];
}
return indi;
}
function isSimilarName(name1: string, name2: string) {
return StringUtils.compareSimilarityPercent(name1, name2) >= 75;
}
function getMarriedName(person: Person) {
if (
!person.Spouses ||
person.LastNameCurrent === 'Unknown' ||
person.LastNameCurrent === person.LastNameAtBirth
) {
return undefined;
}
const nameParts = person.LastNameCurrent.split(/[- ,]/);
// In some languages the same names can differ a bit between genders,
// so regular equals comparison cannot be used.
// To verify if spouse has the same name, person name is split to include
// people with double names, then there is a check if any name part is
// at least 75% similar to spouse name.
const matchingNames = Object.entries(person.Spouses)
.flatMap(([, spousePerson]) => spousePerson.LastNameAtBirth.split(/[- ,]/))
.some((spousePersonNamePart) =>
nameParts.some((personNamePart) =>
isSimilarName(spousePersonNamePart, personNamePart),
),
);
return matchingNames ? person.LastNameCurrent : undefined;
}
/** /**
* Resolve birth name, married name and aka name with following logic: * Specification options for querying from the WikiTree data source.
* - birth name is always prioritized and is set if exists and is not unknown
* - married name is based on LastNameCurrent and is set if it's different than
* birth name and one of the spouses has it as their birth name
* - aka name is based on LastNameOther and is set if it's different than others
*/ */
function convertPersonNames(person: Person) {
const birth =
person.LastNameAtBirth !== 'Unknown' ? person.LastNameAtBirth : undefined;
const married = getMarriedName(person);
const aka =
person.LastNameOther !== 'Unknown' &&
person.LastNameAtBirth !== person.LastNameOther &&
person.LastNameCurrent !== person.LastNameOther
? person.LastNameOther
: undefined;
return {birth, married, aka};
}
/**
* Parses a date in the format returned by WikiTree and converts in to
* the format defined by Topola.
*/
function parseDate(date: string, dataStatus?: string): DateOrRange | undefined {
if (!date) {
return undefined;
}
const matchedDate = date.match(/(\d\d\d\d)-(\d\d)-(\d\d)/);
if (!matchedDate) {
return {date: {text: date}};
}
const parsedDate: Date = {};
if (matchedDate[1] !== '0000') {
parsedDate.year = ~~matchedDate[1];
}
if (matchedDate[2] !== '00') {
parsedDate.month = ~~matchedDate[2];
}
if (matchedDate[3] !== '00') {
parsedDate.day = ~~matchedDate[3];
}
if (dataStatus === 'after') {
return {dateRange: {from: parsedDate}};
}
if (dataStatus === 'before') {
return {dateRange: {to: parsedDate}};
}
if (dataStatus === 'guess') {
parsedDate.qualifier = 'abt';
}
return {date: parsedDate};
}
function parseDecade(decade: string): DateOrRange | undefined {
return decade !== 'unknown' ? {date: {text: decade}} : undefined;
}
const MONTHS = new Map<number, string>([
[1, 'JAN'],
[2, 'FEB'],
[3, 'MAR'],
[4, 'APR'],
[5, 'MAY'],
[6, 'JUN'],
[7, 'JUL'],
[8, 'AUG'],
[9, 'SEP'],
[10, 'OCT'],
[11, 'NOV'],
[12, 'DEC'],
]);
function dateToGedcom(date: Date): string {
return [date.qualifier, date.day, MONTHS.get(date.month!), date.year]
.filter((x) => x !== undefined)
.join(' ');
}
function dateOrRangeToGedcom(dateOrRange: DateOrRange): string {
if (dateOrRange.date) {
return dateToGedcom(dateOrRange.date);
}
if (!dateOrRange.dateRange) {
return '';
}
if (dateOrRange.dateRange.from && dateOrRange.dateRange.to) {
return `BET ${dateToGedcom(dateOrRange.dateRange.from)} AND ${
dateOrRange.dateRange.to
}`;
}
if (dateOrRange.dateRange.from) {
return `AFT ${dateToGedcom(dateOrRange.dateRange.from)}`;
}
if (dateOrRange.dateRange.to) {
return `BEF ${dateToGedcom(dateOrRange.dateRange.to)}`;
}
return '';
}
function nameToGedcom(type: string, firstName?: string, lastName?: string) {
return {
level: 1,
pointer: '',
tag: 'NAME',
data: `${firstName || ''} /${lastName || ''}/`,
tree: [
{
level: 2,
pointer: '',
tag: 'TYPE',
data: type,
tree: [],
},
],
};
}
function eventToGedcom(event: JsonEvent): GedcomEntry[] {
const result = [];
if (isValidDateOrRange(event)) {
result.push({
level: 2,
pointer: '',
tag: 'DATE',
data: dateOrRangeToGedcom(event),
tree: [],
});
}
if (event.place) {
result.push({
level: 2,
pointer: '',
tag: 'PLAC',
data: event.place,
tree: [],
});
}
return result;
}
function imageToGedcom(
image: JsonImage,
fullSizePhotoUrl: string | undefined,
): GedcomEntry[] {
return [
{
level: 2,
pointer: '',
tag: 'FILE',
data: fullSizePhotoUrl || image.url,
tree: [
{
level: 3,
pointer: '',
tag: 'FORM',
data: image.title?.split('.').pop() || '',
tree: [],
},
{
level: 3,
pointer: '',
tag: 'TITL',
data: image.title?.split('.')[0] || '',
tree: [],
},
],
},
];
}
function indiToGedcom(
indi: JsonIndi,
fullSizePhotoUrl: Map<string, string>,
personNames: {birth?: string; married?: string; aka?: string},
): GedcomEntry {
// WikiTree URLs replace spaces with underscores.
const escapedId = indi.id.replace(/ /g, '_');
const record: GedcomEntry = {
level: 0,
pointer: `@${indi.id}@`,
tag: 'INDI',
data: '',
tree: [],
};
if (personNames.birth) {
record.tree.push(nameToGedcom('birth', indi.firstName, personNames.birth));
}
if (personNames.married) {
record.tree.push(
nameToGedcom('married', indi.firstName, personNames.married),
);
}
if (personNames.aka) {
record.tree.push(nameToGedcom('aka', indi.firstName, personNames.aka));
}
if (indi.birth) {
record.tree.push({
level: 1,
pointer: '',
tag: 'BIRT',
data: '',
tree: eventToGedcom(indi.birth),
});
}
if (indi.death) {
record.tree.push({
level: 1,
pointer: '',
tag: 'DEAT',
data: '',
tree: eventToGedcom(indi.death),
});
}
if (indi.famc) {
record.tree.push({
level: 1,
pointer: '',
tag: 'FAMC',
data: `@${indi.famc}@`,
tree: [],
});
}
(indi.fams || []).forEach((fams) =>
record.tree.push({
level: 1,
pointer: '',
tag: 'FAMS',
data: `@${fams}@`,
tree: [],
}),
);
if (!indi.id.startsWith('~')) {
record.tree.push({
level: 1,
pointer: '',
tag: 'WWW',
data: `https://www.wikitree.com/wiki/${escapedId}`,
tree: [],
});
}
(indi.images || []).forEach((image) => {
record.tree.push({
level: 1,
pointer: '',
tag: 'OBJE',
data: '',
tree: imageToGedcom(image, fullSizePhotoUrl.get(indi.id)),
});
});
return record;
}
function famToGedcom(fam: JsonFam): GedcomEntry {
const record: GedcomEntry = {
level: 0,
pointer: `@${fam.id}@`,
tag: 'FAM',
data: '',
tree: [],
};
if (fam.wife) {
record.tree.push({
level: 1,
pointer: '',
tag: 'WIFE',
data: `@${fam.wife}@`,
tree: [],
});
}
if (fam.husb) {
record.tree.push({
level: 1,
pointer: '',
tag: 'HUSB',
data: `@${fam.husb}@`,
tree: [],
});
}
(fam.children || []).forEach((child) =>
record.tree.push({
level: 1,
pointer: child,
tag: 'CHILD',
data: '',
tree: [],
}),
);
if (fam.marriage) {
record.tree.push({
level: 1,
pointer: '',
tag: 'MARR',
data: '',
tree: eventToGedcom(fam.marriage),
});
}
return record;
}
/**
* Creates a GEDCOM structure for the purpose of displaying the details
* panel.
*/
function buildGedcom(
data: JsonGedcomData,
fullSizePhotoUrls: Map<string, string>,
personNames: Map<string, {birth?: string; married?: string; aka?: string}>,
): GedcomData {
const gedcomIndis: {[key: string]: GedcomEntry} = {};
const gedcomFams: {[key: string]: GedcomEntry} = {};
data.indis.forEach((indi) => {
gedcomIndis[indi.id] = indiToGedcom(
indi,
fullSizePhotoUrls,
personNames.get(indi.id) || {},
);
});
data.fams.forEach((fam) => {
gedcomFams[fam.id] = famToGedcom(fam);
});
return {
head: {level: 0, pointer: '', tag: 'HEAD', data: '', tree: []},
indis: gedcomIndis,
fams: gedcomFams,
other: {},
};
}
/**
* Returns a set which is a value from a SetMultimap. If the key doesn't exist,
* an empty set is added to the map.
*/
function getSet<K, V>(map: Map<K, Set<V>>, key: K): Set<V> {
const set = map.get(key);
if (set) {
return set;
}
const newSet = new Set<V>();
map.set(key, newSet);
return newSet;
}
export interface WikiTreeSourceSpec { export interface WikiTreeSourceSpec {
source: DataSourceEnum.WIKITREE; source: DataSourceEnum.WIKITREE;
authcode?: string; authcode?: string;

251
src/datasource/wikitree_api.ts Normal file
View File

@@ -0,0 +1,251 @@
import {
clientLogin,
getLoggedInUserName,
getPeople,
getRelatives as getRelativesApi,
Person,
} from 'wikitree-js';
import {TopolaError} from '../util/error';
const WIKITREE_APP_ID = 'topola-viewer';
/** Prefix for IDs of private individuals. */
export const PRIVATE_ID_PREFIX = '~Private';
const ANCESTORS_GENERATION_LIMIT = 5;
const DESCENDANT_GENERATION_LIMIT = 5;
function getSessionStorageItem(key: string): string | null {
try {
return sessionStorage.getItem(key);
} catch (e) {
console.warn('Failed to load data from session storage: ' + e);
}
return null;
}
function setSessionStorageItem(key: string, value: string) {
try {
return sessionStorage.setItem(key, value);
} catch (e) {
console.warn('Failed to store data in session storage: ' + e);
}
}
function getCacheItem<T>(key: string): T | null {
const cachedData = getSessionStorageItem(key);
if (cachedData) {
try {
return JSON.parse(cachedData) as T;
} catch (e) {
console.warn(`Failed to parse cached data for key ${key}: ${e}`);
}
}
return null;
}
function setCacheItem<T>(key: string, value: T): void {
setSessionStorageItem(key, JSON.stringify(value));
}
function getApiOptions(handleCors: boolean) {
return Object.assign(
{appId: WIKITREE_APP_ID},
handleCors
? {
apiUrl:
'https://topolaproxy.bieda.it/https://api.wikitree.com/api.php',
}
: {},
);
}
async function getAncestors(
key: string,
handleCors: boolean,
): Promise<Person[]> {
const cacheKey = `wikitree:ancestors:${key}`;
const cachedData = getCacheItem<Person[]>(cacheKey);
if (cachedData) {
return cachedData;
}
const result = await getPeople(
[key],
{ancestors: ANCESTORS_GENERATION_LIMIT},
getApiOptions(handleCors),
);
setCacheItem(cacheKey, result);
return result;
}
async function getRelatives(
keys: string[],
handleCors: boolean,
): Promise<Person[]> {
const result: Person[] = [];
const keysToFetch: string[] = [];
keys.forEach((key) => {
const cachedData = getCacheItem<Person>(`wikitree:relatives:${key}`);
if (cachedData) {
result.push(cachedData);
} else {
keysToFetch.push(key);
}
});
if (keysToFetch.length === 0) {
return result;
}
const response = await getRelativesApi(
keysToFetch,
{getChildren: true, getSpouses: true},
getApiOptions(handleCors),
);
if (!response) {
const id = keysToFetch[0];
throw new TopolaError(
'WIKITREE_PROFILE_NOT_FOUND',
`WikiTree profile ${id} not found`,
{id},
);
}
response.forEach((person) => {
setCacheItem(`wikitree:relatives:${person.Name}`, person);
});
return result.concat(response);
}
async function logInIfNeeded(
authcode: string | undefined,
handleCors: boolean,
) {
if (!handleCors && !getLoggedInUserName() && authcode) {
const loginResult = await clientLogin(authcode, {appId: WIKITREE_APP_ID});
if (loginResult.result === 'Success') {
sessionStorage.clear();
}
}
}
async function getFirstPerson(key: string, handleCors: boolean) {
const person = (await getRelatives([key], handleCors))[0];
if (!person?.Name) {
const id = key;
throw new TopolaError(
'WIKITREE_PROFILE_NOT_ACCESSIBLE',
`WikiTree profile ${id} is not accessible. Try logging in.`,
{id},
);
}
return person;
}
function getSpouseKeys(person: Person) {
return Object.values(person.Spouses || {}).map((s) => s.Name);
}
async function getAllAncestors(keys: string[], handleCors: boolean) {
const ancestors = await Promise.all(
keys.map((key) => getAncestors(key, handleCors)),
);
const ancestorKeys = ancestors
.flat()
.map((person) => person.Name)
.filter((key) => !!key);
const ancestorDetails = await getRelatives(ancestorKeys, handleCors);
// Map from person id to father id if the father profile is private.
const privateFathers: Map<number, number> = new Map();
// Map from person id to mother id if the mother profile is private.
const privateMothers: Map<number, number> = new Map();
// Adjust private individual ids so that there are no collisions in the case
// that ancestors were collected for more than one person.
ancestors.forEach((ancestorList, index) => {
const offset = 1000 * index;
// Adjust ids by offset.
ancestorList.forEach((person) => {
if (person.Id < 0) {
person.Id -= offset;
person.Name = `${PRIVATE_ID_PREFIX}${person.Id}`;
}
if (person.Father < 0) {
person.Father -= offset;
privateFathers.set(person.Id, person.Father);
}
if (person.Mother < 0) {
person.Mother -= offset;
privateMothers.set(person.Id, person.Mother);
}
});
});
// Set the Father and Mother fields again because getRelatives doesn't return
// private parents.
ancestorDetails.forEach((person) => {
const privateFather = privateFathers.get(person.Id);
if (privateFather) {
person.Father = privateFather;
}
const privateMother = privateMothers.get(person.Id);
if (privateMother) {
person.Mother = privateMother;
}
});
// Collect private individuals.
const privateAncestors = ancestors.flat().filter((person) => person.Id < 0);
return ancestorDetails.concat(privateAncestors);
}
async function getAllDescendants(key: string, handleCors: boolean) {
const everyone: Person[] = [];
// Fetch descendants recursively.
let toFetch = [key];
let generation = 0;
while (toFetch.length > 0 && generation <= DESCENDANT_GENERATION_LIMIT) {
const people = await getRelatives(toFetch, handleCors);
everyone.push(...people);
const allSpouses = people.flatMap((person) =>
Object.values(person.Spouses || {}),
);
everyone.push(...allSpouses);
// Fetch all children.
toFetch = people.flatMap((person) =>
Object.values(person.Children || {}).map((c) => c.Name),
);
generation++;
}
return everyone;
}
/**
* Loads data from the WikiTree API for a given person key.
*
* @param key The WikiTree profile ID to load.
* @param authcode Optional authentication code.
* @returns A unique list of WikiTree `Person` records.
*/
export async function loadData(key: string, authcode?: string): Promise<Person[]> {
// Work around CORS if not in apps.wikitree.com domain.
const handleCors = window.location.hostname !== 'apps.wikitree.com';
await logInIfNeeded(authcode, handleCors);
const firstPerson = await getFirstPerson(key, handleCors);
const spouseKeys = getSpouseKeys(firstPerson);
// Fetch the ancestors of the input person and ancestors of his/her spouses.
const allAncestors = getAllAncestors([key].concat(spouseKeys), handleCors);
// Fetch descendants and their spouses.
const allDescendants = getAllDescendants(key, handleCors);
const everyone: Person[] = [
...(await allAncestors),
...(await allDescendants),
];
// Make sure the list contains unique elements.
return Array.from(
new Map(everyone.map((person) => [person.Id, person])).values(),
);
}

318
src/datasource/wikitree_transformer.ts Normal file
View File

@@ -0,0 +1,318 @@
import {IntlShape} from 'react-intl';
import {
DateOrRange,
JsonEvent,
JsonFam,
JsonImage,
JsonIndi,
} from 'topola';
import {StringUtils} from 'turbocommons-ts';
import {Person} from 'wikitree-js';
import {PRIVATE_ID_PREFIX} from './wikitree_api';
function getFamilyId(spouse1: number, spouse2: number) {
if (spouse2 > spouse1) {
return `${spouse1}_${spouse2}`;
}
return `${spouse2}_${spouse1}`;
}
function getFamilies(people: Person[]) {
// Map from person id to the set of families where they are a spouse.
const families = new Map<number, Set<string>>();
people.forEach((person) => {
if (person.Mother || person.Father) {
const famId = getFamilyId(person.Mother, person.Father);
getSet(families, person.Mother).add(famId);
getSet(families, person.Father).add(famId);
}
if (person.Spouses) {
Object.values(person.Spouses).forEach((spouse) => {
const famId = getFamilyId(person.Id, spouse.Id);
getSet(families, person.Id).add(famId);
getSet(families, spouse.Id).add(famId);
});
}
});
return families;
}
function getChildren(people: Person[]) {
// Map from family id to the set of children.
const children = new Map<string, Set<number>>();
people.forEach((person) => {
if (person.Mother || person.Father) {
const famId = getFamilyId(person.Mother, person.Father);
getSet(children, famId).add(person.Id);
}
});
return children;
}
function getSpouses(people: Person[]) {
// Map from famliy id to the spouses.
const spouses = new Map<
string,
{wife?: number; husband?: number; spouse?: Person}
>();
people.forEach((person) => {
if (person.Mother || person.Father) {
const famId = getFamilyId(person.Mother, person.Father);
spouses.set(famId, {
wife: person.Mother || undefined,
husband: person.Father || undefined,
});
}
if (person.Spouses) {
Object.values(person.Spouses).forEach((spouse) => {
const famId = getFamilyId(person.Id, spouse.Id);
const familySpouses =
person.Gender === 'Male'
? {wife: spouse.Id, husband: person.Id, spouse}
: {wife: person.Id, husband: spouse.Id, spouse};
spouses.set(famId, familySpouses);
});
}
});
return spouses;
}
/**
* Converts a list of WikiTree Person records into Topola individual records.
*
* @param people List of Person records to convert.
* @param intl Intl shape for localization.
* @returns List of JsonIndi objects.
*/
export function convertIndis(people: Person[], intl: IntlShape): JsonIndi[] {
const families = getFamilies(people);
return people.map((person) => {
const indi = convertPerson(person, intl);
indi.fams = Array.from(getSet(families, person.Id));
return indi;
});
}
/**
* Converts relationships of a list of WikiTree Person records into Topola family records.
*
* @param people List of Person records to convert.
* @returns List of JsonFam objects.
*/
export function convertFams(people: Person[]): JsonFam[] {
// Map from numerical id to human-readable id.
const idToName = new Map(people.map((person) => [person.Id, person.Name]));
const children = getChildren(people);
const spouses = getSpouses(people);
return Array.from(spouses.entries()).map(([key, value]) => {
const fam: JsonFam = {
id: key,
};
const wife = value.wife && idToName.get(value.wife);
if (wife) {
fam.wife = wife;
}
const husband = value.husband && idToName.get(value.husband);
if (husband) {
fam.husb = husband;
}
fam.children = Array.from(getSet(children, key)).map(
(child) => idToName.get(child)!,
);
if (
value.spouse &&
((value.spouse.marriage_date &&
value.spouse.marriage_date !== '0000-00-00') ||
value.spouse.marriage_location)
) {
const parsedDate = parseDate(value.spouse.marriage_date);
fam.marriage = Object.assign({}, parsedDate, {
place: value.spouse.marriage_location,
});
}
return fam;
});
}
function extractNames(person: Person, intl: IntlShape): Partial<JsonIndi> {
const result: Partial<JsonIndi> = {};
if (person.Name.startsWith(PRIVATE_ID_PREFIX)) {
result.hideId = true;
result.firstName = intl.formatMessage({
id: 'wikitree.private',
defaultMessage: 'Private',
});
}
if (person.FirstName && person.FirstName !== 'Unknown') {
result.firstName = person.FirstName;
} else if (person.RealName && person.RealName !== 'Unknown') {
result.firstName = person.RealName;
}
if (person.LastNameAtBirth !== 'Unknown') {
result.lastName = person.LastNameAtBirth;
}
return result;
}
function extractDates(person: Person): {
birth?: JsonEvent;
death?: JsonEvent;
} {
const result: {birth?: JsonEvent; death?: JsonEvent} = {};
if (
(person.BirthDate && person.BirthDate !== '0000-00-00') ||
person.BirthLocation ||
person.BirthDateDecade !== 'unknown'
) {
const parsedDate = parseDate(
person.BirthDate,
(person.DataStatus && person.DataStatus.BirthDate) || undefined,
);
const date = parsedDate || parseDecade(person.BirthDateDecade);
result.birth = Object.assign({}, date, {place: person.BirthLocation});
}
if (
(person.DeathDate && person.DeathDate !== '0000-00-00') ||
person.DeathLocation ||
person.DeathDateDecade !== 'unknown'
) {
const parsedDate = parseDate(
person.DeathDate,
(person.DataStatus && person.DataStatus.DeathDate) || undefined,
);
const date = parsedDate || parseDecade(person.DeathDateDecade);
result.death = Object.assign({}, date, {place: person.DeathLocation});
}
return result;
}
function extractImages(person: Person): JsonImage[] | undefined {
if (person.PhotoData) {
return [
{
url: `https://www.wikitree.com${person.PhotoData.url}`,
title: person.Photo,
},
];
}
return undefined;
}
function convertPerson(person: Person, intl: IntlShape): JsonIndi {
const indi: JsonIndi = Object.assign(
{id: person.Name},
extractNames(person, intl),
extractDates(person),
);
if (person.Mother || person.Father) {
indi.famc = getFamilyId(person.Mother, person.Father);
}
if (person.Gender === 'Male') {
indi.sex = 'M';
} else if (person.Gender === 'Female') {
indi.sex = 'F';
}
const images = extractImages(person);
if (images) {
indi.images = images;
}
return indi;
}
function isSimilarName(name1: string, name2: string) {
return StringUtils.compareSimilarityPercent(name1, name2) >= 75;
}
function getMarriedName(person: Person) {
if (
!person.Spouses ||
person.LastNameCurrent === 'Unknown' ||
person.LastNameCurrent === person.LastNameAtBirth
) {
return undefined;
}
const nameParts = person.LastNameCurrent.split(/[- ,]/);
// In some languages the same names can differ a bit between genders,
// so regular equals comparison cannot be used.
// To verify if spouse has the same name, person name is split to include
// people with double names, then there is a check if any name part is
// at least 75% similar to spouse name.
const matchingNames = Object.entries(person.Spouses)
.flatMap(([, spousePerson]) => spousePerson.LastNameAtBirth.split(/[- ,]/))
.some((spousePersonNamePart) =>
nameParts.some((personNamePart) =>
isSimilarName(spousePersonNamePart, personNamePart),
),
);
return matchingNames ? person.LastNameCurrent : undefined;
}
/**
* Resolves the birth, married, and aka names of a WikiTree Person.
*
* @param person The WikiTree Person to resolve names for.
* @returns Object containing birth, married, and aka names.
*/
export function convertPersonNames(
person: Person,
): {birth?: string; married?: string; aka?: string} {
const birth =
person.LastNameAtBirth !== 'Unknown' ? person.LastNameAtBirth : undefined;
const married = getMarriedName(person);
const aka =
person.LastNameOther !== 'Unknown' &&
person.LastNameAtBirth !== person.LastNameOther &&
person.LastNameCurrent !== person.LastNameOther
? person.LastNameOther
: undefined;
return {birth, married, aka};
}
function parseDate(
date: string,
dataStatus?: string,
): DateOrRange | undefined {
if (!date) {
return undefined;
}
const matchedDate = date.match(/(\d\d\d\d)-(\d\d)-(\d\d)/);
if (!matchedDate) {
return {date: {text: date}};
}
const parsedDate: any = {};
if (matchedDate[1] !== '0000') {
parsedDate.year = ~~matchedDate[1];
}
if (matchedDate[2] !== '00') {
parsedDate.month = ~~matchedDate[2];
}
if (matchedDate[3] !== '00') {
parsedDate.day = ~~matchedDate[3];
}
if (dataStatus === 'after') {
return {dateRange: {from: parsedDate}};
}
if (dataStatus === 'before') {
return {dateRange: {to: parsedDate}};
}
if (dataStatus === 'guess') {
parsedDate.qualifier = 'abt';
}
return {date: parsedDate};
}
function parseDecade(decade: string): DateOrRange | undefined {
return decade !== 'unknown' ? {date: {text: decade}} : undefined;
}
function getSet<K, V>(map: Map<K, Set<V>>, key: K): Set<V> {
const set = map.get(key);
if (set) {
return set;
}
const newSet = new Set<V>();
map.set(key, newSet);
return newSet;
}