Joe Attardi
4 years ago
11 changed files with 4640 additions and 70 deletions
File diff suppressed because it is too large
@ -1,62 +1,142 @@ |
|||||
const { writeFileSync } = require('fs'); |
const fs = require('fs'); |
||||
|
const readline = require('readline'); |
||||
|
|
||||
const rawData = require('emoji-datasource'); |
const DATA_LINE_REGEX = /((?:[0-9A-F]+ ?)+)\s+;.+#.+E([0-9.]+) ([\w\s:,-]+)/; |
||||
|
const EMOJI_WITH_MODIFIER_REGEX = /([a-z]+): ([a-z -]+)/; |
||||
function getEmoji(unified) { |
const EMOJI_WITH_SKIN_TONE_AND_MODIFIER_REGEX = /([a-z]+): ([a-z -]+), ([a-z ]+)/; |
||||
const chars = unified.split('-'); |
|
||||
const codePoints = chars.map(char => parseInt(char, 16)); |
|
||||
return String.fromCodePoint(...codePoints); |
|
||||
} |
|
||||
|
|
||||
const categoryKeys = { |
const categoryKeys = { |
||||
'Smileys & Emotion': 'smileys', |
'Smileys & Emotion': 'smileys', |
||||
'People & Body': 'smileys', |
'People & Body': 'people', |
||||
'Animals & Nature': 'animals', |
'Animals & Nature': 'animals', |
||||
'Food & Drink': 'food', |
'Food & Drink': 'food', |
||||
Activities: 'activities', |
|
||||
'Travel & Places': 'travel', |
'Travel & Places': 'travel', |
||||
Objects: 'objects', |
'Activities': 'activities', |
||||
Symbols: 'symbols', |
'Objects': 'objects', |
||||
Flags: 'flags', |
'Symbols': 'symbols', |
||||
'Skin Tones': 'skinTones' |
'Flags': 'flags' |
||||
}; |
}; |
||||
|
|
||||
const smileysCount = rawData.filter(e => e.category === 'Smileys & Emotion').length; |
const BLACKLIST = [ |
||||
rawData.filter(e => e.category === 'People & Body').forEach(e => e.sort_order += smileysCount); |
'light skin tone', |
||||
|
'medium-light skin tone', |
||||
|
'medium skin tone', |
||||
|
'medium-dark skin tone', |
||||
|
'dark skin tone', |
||||
|
'red hair', |
||||
|
'white hair', |
||||
|
'curly hair', |
||||
|
'bald' |
||||
|
]; |
||||
|
|
||||
const categories = []; |
const MODIFIER_SUBSTITUTIONS = { |
||||
|
'bald': 'no hair' |
||||
|
}; |
||||
|
|
||||
rawData.sort((e1, e2) => e1.sort_order - e2.sort_order); |
const stream = fs.createReadStream('emoji-test.txt'); |
||||
|
|
||||
const newEmojiData = rawData.map(emojiItem => { |
const interface = readline.createInterface(stream); |
||||
let categoryIndex = categories.indexOf(emojiItem.category); |
|
||||
if (categoryIndex < 0) { |
let currentGroup; |
||||
categories.push(emojiItem.category); |
let currentSubgroup; |
||||
categoryIndex = categories.length - 1; |
let categoryIndex; |
||||
} |
|
||||
|
|
||||
const newData = { |
const data = { |
||||
n: emojiItem.short_names, |
categories: [], |
||||
e: getEmoji(emojiItem.unified), |
emoji: [] |
||||
c: categoryIndex, |
}; |
||||
ver: emojiItem.added_in |
|
||||
}; |
interface.on('line', line => { |
||||
|
if (line.startsWith('# group:')) { |
||||
if (emojiItem.skin_variations) { |
currentGroup = line.slice('# group: '.length); |
||||
newData.v = {}; |
if (currentGroup !== 'Component') { |
||||
Object.keys(emojiItem.skin_variations).forEach(variation => { |
data.categories.push(categoryKeys[currentGroup]); |
||||
newData.v[variation] = { |
categoryIndex = data.categories.length - 1; |
||||
k: `${emojiItem.short_names[0]}-${variation}`, |
} |
||||
n: emojiItem.short_names[0], |
} else if (line.startsWith('# subgroup:')) { |
||||
e: getEmoji(emojiItem.skin_variations[variation].unified) |
currentSubgroup = line.slice('# subgroup: '.length); |
||||
}; |
} else if (!line.startsWith('#') && currentGroup !== 'Component') { |
||||
}); |
const matcher = DATA_LINE_REGEX.exec(line); |
||||
|
if (matcher) { |
||||
|
const sequence = matcher[1].trim(); |
||||
|
const emoji = getEmoji(sequence); |
||||
|
let name = matcher[3]; |
||||
|
|
||||
|
let version = matcher[2]; |
||||
|
console.log(version); |
||||
|
if (version === '0.6' || version === '0.7') { |
||||
|
version = '1.0'; |
||||
|
} |
||||
|
|
||||
|
if (currentSubgroup === 'person') { |
||||
|
const modifierMatcher = EMOJI_WITH_MODIFIER_REGEX.exec(name); |
||||
|
const skinToneMatcher = EMOJI_WITH_SKIN_TONE_AND_MODIFIER_REGEX.exec(name); |
||||
|
if (skinToneMatcher) { |
||||
|
name = skinToneMatcher[1] + ' with ' + substituteModifier(skinToneMatcher[3]) + ': ' + skinToneMatcher[2]; |
||||
|
} else if (modifierMatcher) { |
||||
|
if (!modifierMatcher[2].includes('skin tone')) { |
||||
|
name = modifierMatcher[1] + ' with ' + substituteModifier(modifierMatcher[2]); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
data.emoji.push({ sequence, emoji, category: categoryIndex, name, variations: [], version }); |
||||
|
} |
||||
} |
} |
||||
|
}); |
||||
|
|
||||
return newData; |
interface.on('close', () => { |
||||
|
stream.close(); |
||||
|
|
||||
|
let toDelete = []; |
||||
|
|
||||
|
const emojisWithVariationSelector = data.emoji.filter(emoji => emoji.sequence.includes('FE0F')); |
||||
|
emojisWithVariationSelector.forEach(emoji => { |
||||
|
const baseEmoji = data.emoji.find(e => e.sequence === emoji.sequence.replace(' FE0F', '')); |
||||
|
toDelete.push(baseEmoji); |
||||
|
}); |
||||
|
|
||||
|
data.emoji = data.emoji.filter(e => !toDelete.includes(e)); |
||||
|
toDelete = []; |
||||
|
|
||||
|
BLACKLIST.forEach(name => toDelete.push(data.emoji.find(e => e.name === name))); |
||||
|
|
||||
|
const emojisWithVariations = data.emoji.filter(emoji => emoji.name.includes(':') && !emoji.name.startsWith('family')); |
||||
|
emojisWithVariations.forEach(emoji => { |
||||
|
const baseName = emoji.name.split(':')[0]; |
||||
|
const baseEmoji = data.emoji.find(e => e.name === baseName); |
||||
|
if (baseEmoji) { |
||||
|
baseEmoji.variations.push(emoji.emoji); |
||||
|
toDelete.push(emoji); |
||||
|
} |
||||
|
}); |
||||
|
|
||||
|
// Cleanup
|
||||
|
data.emoji = data.emoji.filter(e => !toDelete.includes(e)); |
||||
|
data.emoji.forEach(emoji => { |
||||
|
delete emoji.sequence; |
||||
|
if (!emoji.variations.length) { |
||||
|
delete emoji.variations; |
||||
|
} |
||||
|
}); |
||||
|
|
||||
|
fs.writeFileSync('emoji.js', `export default ${JSON.stringify(data)}`); |
||||
}); |
}); |
||||
|
|
||||
writeFileSync( |
function getEmoji(sequence) { |
||||
'src/data/emoji.js', |
const chars = sequence.split(' '); |
||||
`export default { categories: ${JSON.stringify(categories.map(category => categoryKeys[category]))}, emojiData: ${JSON.stringify(newEmojiData)}}` |
const codePoints = chars.map(char => parseInt(char, 16)); |
||||
); |
return String.fromCodePoint(...codePoints); |
||||
|
} |
||||
|
|
||||
|
function substituteModifier(name) { |
||||
|
const substitutions = Object.keys(MODIFIER_SUBSTITUTIONS); |
||||
|
for (let i = 0; i < substitutions.length; i++) { |
||||
|
const substitution = substitutions[i]; |
||||
|
if (name.includes(substitution)) { |
||||
|
return name.replace(substitution, MODIFIER_SUBSTITUTIONS[substitution]); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return name; |
||||
|
} |
Loading…
Reference in new issue