Refactoring: moved some source files into subdirectories

This commit is contained in:
Przemek Wiech
2020-05-03 20:53:59 +02:00
parent f588c62696
commit a61b4848a5
14 changed files with 29 additions and 31 deletions

View File

@@ -0,0 +1,92 @@
import {convertGedcom, TopolaData} from '../util/gedcom_util';
import {IndiInfo, JsonGedcomData} from 'topola';
/**
* Returns a valid IndiInfo object, either with the given indi and generation
* or with an individual taken from the data and generation 0.
*/
export function getSelection(
data: JsonGedcomData,
indi?: string,
generation?: number,
): IndiInfo {
// If ID is not given or it doesn't exist in the data, use the first ID in
// the data.
const id =
indi && data.indis.some((i) => i.id === indi) ? indi : data.indis[0].id;
return {id, generation: generation || 0};
}
function prepareData(
gedcom: string,
cacheId: string,
images?: Map<string, string>,
): TopolaData {
const data = convertGedcom(gedcom, images || new Map());
const serializedData = JSON.stringify(data);
try {
sessionStorage.setItem(cacheId, serializedData);
} catch (e) {
console.warn('Failed to store data in session storage: ' + e);
}
return data;
}
/** Fetches data from the given URL. Uses cors-anywhere if handleCors is true. */
export async function loadFromUrl(
url: string,
handleCors: boolean,
): Promise<TopolaData> {
try {
const cachedData = sessionStorage.getItem(url);
if (cachedData) {
return JSON.parse(cachedData);
}
} catch (e) {
console.warn('Failed to load data from session storage: ' + e);
}
const driveUrlMatch1 = url.match(
/https:\/\/drive\.google\.com\/file\/d\/(.*)\/.*/,
);
if (driveUrlMatch1) {
url = `https://drive.google.com/uc?id=${driveUrlMatch1[1]}&export=download`;
}
const driveUrlMatch2 = url.match(
/https:\/\/drive\.google\.com\/open\?id=([^&]*)&?.*/,
);
if (driveUrlMatch2) {
url = `https://drive.google.com/uc?id=${driveUrlMatch2[1]}&export=download`;
}
const urlToFetch = handleCors
? 'https://cors-anywhere.herokuapp.com/' + url
: url;
const response = await window.fetch(urlToFetch);
if (response.status !== 200) {
throw new Error(response.statusText);
}
const gedcom = await response.text();
return prepareData(gedcom, url);
}
/** Loads data from the given GEDCOM file contents. */
export async function loadGedcom(
hash: string,
gedcom?: string,
images?: Map<string, string>,
): Promise<TopolaData> {
try {
const cachedData = sessionStorage.getItem(hash);
if (cachedData) {
return JSON.parse(cachedData);
}
} catch (e) {
console.warn('Failed to load data from session storage: ' + e);
}
if (!gedcom) {
throw new Error('Error loading data. Please upload your file again.');
}
return prepareData(gedcom, hash, images);
}

561
src/datasource/wikitree.ts Normal file
View File

@@ -0,0 +1,561 @@
import Cookies from 'js-cookie';
import {Date, DateOrRange, JsonFam, JsonIndi} from 'topola';
import {GedcomData, normalizeGedcom, TopolaData} from '../util/gedcom_util';
import {GedcomEntry} from 'parse-gedcom';
import {InjectedIntl} from 'react-intl';
/** Prefix for IDs of private individuals. */
export const PRIVATE_ID_PREFIX = '~Private';
/**
* Cookie where the logged in user name is stored. This cookie is shared
* between apps hosted on apps.wikitree.com.
*/
const USER_NAME_COOKIE = 'wikidb_wtb_UserName';
/** WikiTree API getAncestors request. */
interface GetAncestorsRequest {
action: 'getAncestors';
key: string;
fields: string;
}
/** WikiTree API getRelatives request. */
interface GetRelativesRequest {
action: 'getRelatives';
keys: string;
getChildren?: true;
getSpouses?: true;
}
/** WikiTree API clientLogin request. */
interface ClientLoginRequest {
action: 'clientLogin';
authcode: string;
}
/** WikiTree API clientLogin response. */
interface ClientLoginResponse {
result: string;
username: string;
}
type WikiTreeRequest =
| GetAncestorsRequest
| GetRelativesRequest
| ClientLoginRequest;
/** Person structure returned from WikiTree API. */
interface Person {
Id: number;
Name: string;
FirstName: string;
LastNameAtBirth: string;
RealName: string;
Spouses: {[key: number]: Person};
Children: {[key: number]: Person};
Mother: number;
Father: number;
Gender: string;
BirthDate: string;
DeathDate: string;
BirthLocation: string;
DeathLocation: string;
BirthDateDecade: string;
DeathDateDecade: string;
marriage_location: string;
marriage_date: string;
DataStatus?: {
BirthDate: string;
DeathDate: string;
};
PhotoData?: {
path: string;
url: string;
};
}
/** Gets item from session storage. Logs exception if one is thrown. */
function getSessionStorageItem(key: string): string | null {
try {
return sessionStorage.getItem(key);
} catch (e) {
console.warn('Failed to load data from session storage: ' + e);
}
return null;
}
/** Sets item in session storage. Logs exception if one is thrown. */
function setSessionStorageItem(key: string, value: string) {
try {
sessionStorage.setItem(key, value);
} catch (e) {
console.warn('Failed to store data in session storage: ' + e);
}
}
/** Sends a request to the WikiTree API. Returns the parsed response JSON. */
async function wikiTreeGet(request: WikiTreeRequest, handleCors: boolean) {
const requestData = new FormData();
requestData.append('format', 'json');
for (const key in request) {
requestData.append(key, request[key]);
}
const apiUrl = handleCors
? 'https://cors-anywhere.herokuapp.com/https://api.wikitree.com/api.php'
: 'https://api.wikitree.com/api.php';
const response = await window.fetch(apiUrl, {
method: 'POST',
body: requestData,
credentials: handleCors ? undefined : 'include',
});
const responseBody = await response.text();
return JSON.parse(responseBody);
}
/**
* Retrieves ancestors from WikiTree for the given person ID.
* Uses sessionStorage for caching responses.
*/
async function getAncestors(
key: string,
handleCors: boolean,
): Promise<Person[]> {
const cacheKey = `wikitree:ancestors:${key}`;
const cachedData = getSessionStorageItem(cacheKey);
if (cachedData) {
return JSON.parse(cachedData);
}
const response = await wikiTreeGet(
{
action: 'getAncestors',
key: key,
fields: '*',
},
handleCors,
);
const result = response[0].ancestors as Person[];
setSessionStorageItem(cacheKey, JSON.stringify(result));
return result;
}
/**
* Retrieves relatives from WikiTree for the given array of person IDs.
* Uses sessionStorage for caching responses.
*/
async function getRelatives(
keys: string[],
handleCors: boolean,
): Promise<Person[]> {
const result: Person[] = [];
const keysToFetch: string[] = [];
keys.forEach((key) => {
const cachedData = getSessionStorageItem(`wikitree:relatives:${key}`);
if (cachedData) {
result.push(JSON.parse(cachedData));
} else {
keysToFetch.push(key);
}
});
if (keysToFetch.length === 0) {
return result;
}
const response = await wikiTreeGet(
{
action: 'getRelatives',
keys: keysToFetch.join(','),
getChildren: true,
getSpouses: true,
},
handleCors,
);
if (response[0].items === null) {
throw new Error(`WikiTree profile ${keysToFetch[0]} not found.`);
}
const fetchedResults = response[0].items.map(
(x: {person: Person}) => x.person,
) as Person[];
fetchedResults.forEach((person) => {
setSessionStorageItem(
`wikitree:relatives:${person.Name}`,
JSON.stringify(person),
);
});
return result.concat(fetchedResults);
}
export async function clientLogin(
authcode: string,
): Promise<ClientLoginResponse> {
const response = await wikiTreeGet(
{
action: 'clientLogin',
authcode,
},
false,
);
return response.clientLogin;
}
/**
* Returnes the logged in user name or undefined if not logged in.
*
* This is not an authoritative answer. The result of this function relies on
* the cookies set on the apps.wikitree.com domain under which this application
* is hosted. The authoritative source of login information is in cookies set on
* the api.wikitree.com domain.
*/
export function getLoggedInUserName(): string | undefined {
return Cookies.get(USER_NAME_COOKIE);
}
/**
* Loads data from WikiTree to populate an hourglass chart starting from the
* given person ID.
*/
export async function loadWikiTree(
key: string,
intl: InjectedIntl,
authcode?: string,
): Promise<TopolaData> {
// Work around CORS if not in apps.wikitree.com domain.
const handleCors = window.location.hostname !== 'apps.wikitree.com';
if (!handleCors && !getLoggedInUserName() && authcode) {
const loginResult = await clientLogin(authcode);
if (loginResult.result === 'Success') {
sessionStorage.clear();
Cookies.set(USER_NAME_COOKIE, loginResult.username);
}
}
const everyone: Person[] = [];
// Fetch the ancestors of the input person and ancestors of his/her spouses.
const firstPerson = await getRelatives([key], handleCors);
if (!firstPerson[0].Name) {
throw new Error(
`WikiTree profile ${key} is not accessible. Try logging in.`,
);
}
const spouseKeys = Object.values(firstPerson[0].Spouses).map((s) => s.Name);
const ancestors = await Promise.all(
[key]
.concat(spouseKeys)
.map((personId) => getAncestors(personId, handleCors)),
);
const ancestorKeys = ancestors
.flat()
.map((person) => person.Name)
.filter((key) => !!key);
const ancestorDetails = await getRelatives(ancestorKeys, handleCors);
// Map from person id to father id if the father profile is private.
const privateFathers: Map<number, number> = new Map();
// Map from person id to mother id if the mother profile is private.
const privateMothers: Map<number, number> = new Map();
// Andujst private individual ids so that there are no collisions in the case
// that ancestors were collected for more than one person.
ancestors.forEach((ancestorList, index) => {
const offset = 1000 * index;
// Adjust ids by offset.
ancestorList.forEach((person) => {
if (person.Id < 0) {
person.Id -= offset;
person.Name = `${PRIVATE_ID_PREFIX}${person.Id}`;
}
if (person.Father < 0) {
person.Father -= offset;
privateFathers.set(person.Id, person.Father);
}
if (person.Mother < 0) {
person.Mother -= offset;
privateMothers.set(person.Id, person.Mother);
}
});
});
// Set the Father and Mother fields again because getRelatives doesn't return
// private parents.
ancestorDetails.forEach((person) => {
const privateFather = privateFathers.get(person.Id);
if (privateFather) {
person.Father = privateFather;
}
const privateMother = privateMothers.get(person.Id);
if (privateMother) {
person.Mother = privateMother;
}
});
everyone.push(...ancestorDetails);
// Collect private individuals.
const privateAncestors = ancestors.flat().filter((person) => person.Id < 0);
everyone.push(...privateAncestors);
// Limit the number of generations of descendants because there may be tens of
// generations for some profiles.
const descendantGenerationLimit = 5;
// Fetch descendants recursively.
let toFetch = [key];
let generation = 0;
while (toFetch.length > 0 && generation <= descendantGenerationLimit) {
const people = await getRelatives(toFetch, handleCors);
everyone.push(...people);
const allSpouses = people.flatMap((person) =>
Object.values(person.Spouses),
);
everyone.push(...allSpouses);
// Fetch all children.
toFetch = people.flatMap((person) =>
Object.values(person.Children).map((c) => c.Name),
);
generation++;
}
// Map from person id to the set of families where they are a spouse.
const families = new Map<number, Set<string>>();
// Map from family id to the set of children.
const children = new Map<string, Set<number>>();
// Map from famliy id to the spouses.
const spouses = new Map<
string,
{wife?: number; husband?: number; spouse?: Person}
>();
// Map from numerical id to human-readable id.
const idToName = new Map<number, string>();
everyone.forEach((person) => {
idToName.set(person.Id, person.Name);
if (person.Mother || person.Father) {
const famId = getFamilyId(person.Mother, person.Father);
getSet(families, person.Mother).add(famId);
getSet(families, person.Father).add(famId);
getSet(children, famId).add(person.Id);
spouses.set(famId, {
wife: person.Mother || undefined,
husband: person.Father || undefined,
});
}
});
const indis: JsonIndi[] = [];
const converted = new Set<number>();
everyone.forEach((person) => {
if (converted.has(person.Id)) {
return;
}
converted.add(person.Id);
const indi = convertPerson(person, intl);
if (person.Spouses) {
Object.values(person.Spouses).forEach((spouse) => {
const famId = getFamilyId(person.Id, spouse.Id);
getSet(families, person.Id).add(famId);
getSet(families, spouse.Id).add(famId);
const familySpouses =
person.Gender === 'Male'
? {wife: spouse.Id, husband: person.Id, spouse}
: {wife: person.Id, husband: spouse.Id, spouse};
spouses.set(famId, familySpouses);
});
}
indi.fams = Array.from(getSet(families, person.Id));
indis.push(indi);
});
const fams = Array.from(spouses.entries()).map(([key, value]) => {
const fam: JsonFam = {
id: key,
};
const wife = value.wife && idToName.get(value.wife);
if (wife) {
fam.wife = wife;
}
const husband = value.husband && idToName.get(value.husband);
if (husband) {
fam.husb = husband;
}
fam.children = Array.from(getSet(children, key)).map(
(child) => idToName.get(child)!,
);
if (
value.spouse &&
((value.spouse.marriage_date &&
value.spouse.marriage_date !== '0000-00-00') ||
value.spouse.marriage_location)
) {
const parsedDate = parseDate(value.spouse.marriage_date);
fam.marriage = Object.assign({}, parsedDate, {
place: value.spouse.marriage_location,
});
}
return fam;
});
const chartData = normalizeGedcom({indis, fams});
const gedcom = buildGedcom(indis);
return {chartData, gedcom};
}
/** Creates a family identifier given 2 spouse identifiers. */
function getFamilyId(spouse1: number, spouse2: number) {
if (spouse2 > spouse1) {
return `${spouse1}_${spouse2}`;
}
return `${spouse2}_${spouse1}`;
}
function convertPerson(person: Person, intl: InjectedIntl): JsonIndi {
const indi: JsonIndi = {
id: person.Name,
};
if (person.Name.startsWith(PRIVATE_ID_PREFIX)) {
indi.hideId = true;
indi.firstName = intl.formatMessage({
id: 'wikitree.private',
defaultMessage: 'Private',
});
}
if (person.FirstName && person.FirstName !== 'Unknown') {
indi.firstName = person.FirstName;
} else if (person.RealName && person.RealName !== 'Unknown') {
indi.firstName = person.RealName;
}
if (person.LastNameAtBirth !== 'Unknown') {
indi.lastName = person.LastNameAtBirth;
}
if (person.Mother || person.Father) {
indi.famc = getFamilyId(person.Mother, person.Father);
}
if (person.Gender === 'Male') {
indi.sex = 'M';
} else if (person.Gender === 'Female') {
indi.sex = 'F';
}
if (
(person.BirthDate && person.BirthDate !== '0000-00-00') ||
person.BirthLocation ||
person.BirthDateDecade !== 'unknown'
) {
const parsedDate = parseDate(
person.BirthDate,
person.DataStatus && person.DataStatus.BirthDate,
);
const date = parsedDate || parseDecade(person.BirthDateDecade);
indi.birth = Object.assign({}, date, {place: person.BirthLocation});
}
if (
(person.DeathDate && person.DeathDate !== '0000-00-00') ||
person.DeathLocation ||
person.DeathDateDecade !== 'unknown'
) {
const parsedDate = parseDate(
person.DeathDate,
person.DataStatus && person.DataStatus.DeathDate,
);
const date = parsedDate || parseDecade(person.DeathDateDecade);
indi.death = Object.assign({}, date, {place: person.DeathLocation});
}
if (person.PhotoData) {
indi.images = [{url: `https://www.wikitree.com${person.PhotoData.url}`}];
}
return indi;
}
/**
* Parses a date in the format returned by WikiTree and converts in to
* the format defined by Topola.
*/
function parseDate(date: string, dataStatus?: string): DateOrRange | undefined {
if (!date) {
return undefined;
}
const matchedDate = date.match(/(\d\d\d\d)-(\d\d)-(\d\d)/);
if (!matchedDate) {
return {date: {text: date}};
}
const parsedDate: Date = {};
if (matchedDate[1] !== '0000') {
parsedDate.year = ~~matchedDate[1];
}
if (matchedDate[2] !== '00') {
parsedDate.month = ~~matchedDate[2];
}
if (matchedDate[3] !== '00') {
parsedDate.day = ~~matchedDate[3];
}
if (dataStatus === 'after') {
return {dateRange: {from: parsedDate}};
}
if (dataStatus === 'before') {
return {dateRange: {to: parsedDate}};
}
if (dataStatus === 'guess') {
parsedDate.qualifier = 'abt';
}
return {date: parsedDate};
}
function parseDecade(decade: string): DateOrRange | undefined {
return decade !== 'unknown' ? {date: {text: decade}} : undefined;
}
/**
* Creates a GEDCOM structure for the purpose of displaying the details
* panel.
*/
function buildGedcom(indis: JsonIndi[]): GedcomData {
const gedcomIndis: {[key: string]: GedcomEntry} = {};
indis.forEach((indi) => {
// WikiTree URLs replace spaces with underscores.
const escapedId = indi.id.replace(/ /g, '_');
gedcomIndis[indi.id] = {
level: 0,
pointer: `@${indi.id}@`,
tag: 'INDI',
data: '',
tree: [
{
level: 1,
pointer: '',
tag: 'NAME',
data: `${indi.firstName || ''} /${indi.lastName || ''}/`,
tree: [],
},
],
};
if (!indi.id.startsWith('~')) {
gedcomIndis[indi.id].tree.push({
level: 1,
pointer: '',
tag: 'WWW',
data: `https://www.wikitree.com/wiki/${escapedId}`,
tree: [],
});
}
});
return {
head: {level: 0, pointer: '', tag: 'HEAD', data: '', tree: []},
indis: gedcomIndis,
fams: {},
other: {},
};
}
/**
* Returns a set which is a value from a SetMultimap. If the key doesn't exist,
* an empty set is added to the map.
*/
function getSet<K, V>(map: Map<K, Set<V>>, key: K): Set<V> {
const set = map.get(key);
if (set) {
return set;
}
const newSet = new Set<V>();
map.set(key, newSet);
return newSet;
}