diff --git a/Makefile b/Makefile index 9217cc4e2..4773371fe 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - EMOJI = NotoColorEmoji font: $(EMOJI).ttf @@ -20,16 +19,32 @@ CFLAGS = -std=c99 -Wall -Wextra `pkg-config --cflags --libs cairo` LDFLAGS = `pkg-config --libs cairo` PNGQUANTDIR := third_party/pngquant PNGQUANT := $(PNGQUANTDIR)/pngquant -PNGQUANTFLAGS = --speed 1 --skip-if-larger --ext '.png' --force +PNGQUANTFLAGS = --speed 1 --skip-if-larger --force -$(PNGQUANT): - $(MAKE) -C $(PNGQUANTDIR) +# zopflipng is better (about 10%) but much slower. it will be used if +# present. pass ZOPFLIPNG= as an arg to make to use optipng instead. -waveflag: waveflag.c - $(CC) $< -o $@ $(CFLAGS) $(LDFLAGS) +ZOPFLIPNG = zopflipng +OPTIPNG = optipng + +EMOJI_BUILDER = third_party/color_emoji/emoji_builder.py +ADD_GLYPHS = third_party/color_emoji/add_glyphs.py +PUA_ADDER = map_pua_emoji.py +VS_ADDER = add_vs_cmap.py # from nototools + +EMOJI_SRC_DIR := png/128 +FLAGS_SRC_DIR := third_party/region-flags/png + +BUILD_DIR := build +EMOJI_DIR := $(BUILD_DIR)/emoji +FLAGS_DIR := $(BUILD_DIR)/flags +RESIZED_FLAGS_DIR := $(BUILD_DIR)/resized_flags +RENAMED_FLAGS_DIR := $(BUILD_DIR)/renamed_flags +QUANTIZED_DIR := $(BUILD_DIR)/quantized_pngs +COMPRESSED_DIR := $(BUILD_DIR)/compressed_pngs LIMITED_FLAGS = CN DE ES FR GB IT JP KR RU US -FLAGS = AD AE AF AG AI AL AM AO AR AS AT AU AW AX AZ \ +SELECTED_FLAGS = AD AE AF AG AI AL AM AO AR AS AT AU AW AX AZ \ BA BB BD BE BF BG BH BI BJ BM BN BO BR BS BT BW BY BZ \ CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ \ DE DJ DK DM DO DZ \ @@ -54,51 +69,127 @@ FLAGS = AD AE AF AG AI AL AM AO AR AS AT AU AW AX AZ \ WS \ YE \ ZA ZM ZW +ALL_FLAGS = $(basename $(notdir $(wildcard $(FLAGS_SRC_DIR)/*.png))) -FLAGS_SRC_DIR = third_party/region-flags/png -FLAGS_DIR = ./flags +FLAGS = $(SELECTED_FLAGS) -GLYPH_NAMES := $(shell ./flag_glyph_name.py $(FLAGS)) -WAVED_FLAGS := $(foreach flag,$(FLAGS),$(FLAGS_DIR)/$(flag).png) -PNG128_FLAGS := $(foreach glyph_name,$(GLYPH_NAMES),$(addprefix ./png/128/emoji_$(glyph_name),.png)) +FLAG_NAMES = $(FLAGS:%=%.png) +FLAG_FILES = $(addprefix $(FLAGS_DIR)/, $(FLAG_NAMES)) +RESIZED_FLAG_FILES = $(addprefix $(RESIZED_FLAGS_DIR)/, $(FLAG_NAMES)) -$(FLAGS_DIR)/%.png: $(FLAGS_SRC_DIR)/%.png ./waveflag $(PNGQUANT) - mkdir -p $(FLAGS_DIR) - ./waveflag "$<" "$@" - optipng -quiet -o7 "$@" - $(PNGQUANT) $(PNGQUANTFLAGS) "$@" +FLAG_GLYPH_NAMES = $(shell ./flag_glyph_name.py $(FLAGS)) +RENAMED_FLAG_NAMES = $(FLAG_GLYPH_NAMES:%=emoji_%.png) +RENAMED_FLAG_FILES = $(addprefix $(RENAMED_FLAGS_DIR)/, $(RENAMED_FLAG_NAMES)) -flag-symlinks: $(WAVED_FLAGS) - $(subst ^, , \ - $(join \ - $(FLAGS:%=ln^-fs^../../flags/%.png^), \ - $(GLYPH_NAMES:%=./png/128/emoji_%.png;) \ - ) \ - ) +EMOJI_NAMES = $(notdir $(wildcard $(EMOJI_SRC_DIR)/emoji_u*.png)) +EMOJI_FILES= $(addprefix $(EMOJI_DIR)/,$(EMOJI_NAMES))) -$(PNG128_FLAGS): flag-symlinks +ALL_NAMES = $(EMOJI_NAMES) $(RENAMED_FLAG_NAMES) -EMOJI_PNG128 = ./png/128/emoji_u +ALL_QUANTIZED_FILES = $(addprefix $(QUANTIZED_DIR)/, $(ALL_NAMES)) +ALL_COMPRESSED_FILES = $(addprefix $(COMPRESSED_DIR)/, $(ALL_NAMES)) -EMOJI_BUILDER = third_party/color_emoji/emoji_builder.py -ADD_GLYPHS = third_party/color_emoji/add_glyphs.py -PUA_ADDER = map_pua_emoji.py -VS_ADDER = add_vs_cmap.py -ifeq (, $(shell which $(VS_ADDER))) - $(error "$(VS_ADDER) not in path, run setup.py in nototools") +# tool checks +ifeq (,$(shell which $(ZOPFLIPNG))) + ifeq (,$(wildcard $(ZOPFLIPNG))) + MISSING_ZOPFLI = fail + endif endif -%.ttx: %.ttx.tmpl $(ADD_GLYPHS) $(UNI) $(PNG128_FLAGS) - python $(ADD_GLYPHS) "$<" "$@" "$(EMOJI_PNG128)" +ifeq (,$(shell which $(OPTIPNG))) + ifeq (,$(wildcard $(OPTIPNG))) + MISSING_OPTIPNG = fail + endif +endif + +ifeq (, $(shell which $(VS_ADDER))) + MISSING_ADDER = fail +endif + + +emoji: $(EMOJI_FILES) + +flags: $(FLAG_FILES) + +resized_flags: $(RESIZED_FLAG_FILES) + +renamed_flags: $(RENAMED_FLAG_FILES) + +quantized: $(ALL_QUANTIZED_FILES) + +compressed: $(ALL_COMPRESSED_FILES) + +check_compress_tool: +ifdef MISSING_ZOPFLI + ifdef MISSING_OPTIPNG + $(error "neither $(ZOPFLIPNG) nor $(OPTIPNG) is available") + else + @echo "using $(OPTIPNG)" + endif +else + @echo "using $(ZOPFLIPNG)" +endif + +check_vs_adder: +ifdef MISSING_ADDER + $(error "$(VS_ADDER) not in path, run setup.py in nototools") +endif + + +$(EMOJI_DIR) $(FLAGS_DIR) $(RESIZED_FLAGS_DIR) $(RENAMED_FLAGS_DIR) $(QUANTIZED_DIR) $(COMPRESSED_DIR): + mkdir -p "$@" + +$(PNGQUANT): + $(MAKE) -C $(PNGQUANTDIR) + +waveflag: waveflag.c + $(CC) $< -o $@ $(CFLAGS) $(LDFLAGS) + +$(EMOJI_DIR)/%.png: $(EMOJI_SRC_DIR)/%.png | $(EMOJI_DIR) + echo "emoji $< $@" + @convert -extent 136x128 -gravity center -background none "$<" "$@" + +$(FLAGS_DIR)/%.png: $(FLAGS_SRC_DIR)/%.png ./waveflag $(PNGQUANT) | $(FLAGS_DIR) + @./waveflag "$<" "$@" + +$(RESIZED_FLAGS_DIR)/%.png: $(FLAGS_DIR)/%.png | $(RESIZED_FLAGS_DIR) + @convert -extent 136x128 -gravity center -background none "$<" "$@" + +flag-symlinks: $(RESIZED_FLAG_FILES) | $(RENAMED_FLAGS_DIR) + @$(subst ^, , \ + $(join \ + $(FLAGS:%=ln^-fs^../resized_flags/%.png^), \ + $(RENAMED_FLAG_FILES:%=%; ) \ + ) \ + ) + +$(RENAMED_FLAG_FILES): flag-symlinks + +$(QUANTIZED_DIR)/%.png: $(RENAMED_FLAGS_DIR)/%.png $(PNGQUANT) | $(QUANTIZED_DIR) + $(PNGQUANT) $(PNGQUANTFLAGS) -o "$@" "$<" + +$(QUANTIZED_DIR)/%.png: $(EMOJI_DIR)/%.png $(PNGQUANT) | $(QUANTIZED_DIR) + $(PNGQUANT) $(PNGQUANTFLAGS) -o "$@" "$<" + +$(COMPRESSED_DIR)/%.png: $(QUANTIZED_DIR)/%.png | check_compress_tool $(COMPRESSED_DIR) +ifdef MISSING_ZOPFLI + $(OPTIPNG) -quiet -o7 -clobber -force -out "$@" "$<" +else + $(ZOPFLIPNG) -y "$<" "$@" 2> /dev/null +endif + + +%.ttx: %.ttx.tmpl $(ADD_GLYPHS) $(ALL_COMPRESSED_FILES) + @python $(ADD_GLYPHS) "$<" "$@" "$(COMPRESSED_DIR)/emoji_u" %.ttf: %.ttx @rm -f "$@" ttx "$<" $(EMOJI).ttf: $(EMOJI).tmpl.ttf $(EMOJI_BUILDER) $(PUA_ADDER) \ - $(EMOJI_PNG128)*.png $(PNG128_FLAGS) - python $(EMOJI_BUILDER) -V $< "$@" $(EMOJI_PNG128) - python $(PUA_ADDER) "$@" "$@-with-pua" + $(ALL_COMPRESSED_FILES) | check_vs_adder + @python $(EMOJI_BUILDER) -V $< "$@" "$(COMPRESSED_DIR)/emoji_u" + @python $(PUA_ADDER) "$@" "$@-with-pua" $(VS_ADDER) --dstdir '.' -o "$@-with-pua-varsel" "$@-with-pua" mv "$@-with-pua-varsel" "$@" rm "$@-with-pua" @@ -106,5 +197,10 @@ $(EMOJI).ttf: $(EMOJI).tmpl.ttf $(EMOJI_BUILDER) $(PUA_ADDER) \ clean: rm -f $(EMOJI).ttf $(EMOJI).tmpl.ttf $(EMOJI).tmpl.ttx rm -f waveflag - rm -rf $(FLAGS_DIR) - rm -f `find -type l -name "*.png"` + rm -rf $(BUILD_DIR) + +.SECONDARY: $(EMOJI_FILES) $(FLAG_FILES) $(RESIZED_FLAG_FILES) $(RENAMED_FLAG_FILES) \ + $(ALL_QUANTIZED_FILES) $(ALL_COMPRESSED_FILES) + +.PHONY: clean flags emoji renamed_flags quantized compressed check_compress_tool + diff --git a/NotoColorEmoji.tmpl.ttx.tmpl b/NotoColorEmoji.tmpl.ttx.tmpl index 64461de00..9ab095006 100644 --- a/NotoColorEmoji.tmpl.ttx.tmpl +++ b/NotoColorEmoji.tmpl.ttx.tmpl @@ -4,12 +4,16 @@ + + + + - + @@ -119,12 +123,19 @@ + + + + + + + @@ -145,7 +156,7 @@ Noto Color Emoji - Version 1.22 + Version 1.23 NotoColorEmoji diff --git a/generate_emoji_placeholders.py b/generate_emoji_placeholders.py new file mode 100644 index 000000000..48b508231 --- /dev/null +++ b/generate_emoji_placeholders.py @@ -0,0 +1,95 @@ +import os +from os import path +import subprocess + +OUTPUT_DIR = '/tmp/placeholder_emoji' + +def generate_image(name, text): + print name, text.replace('\n', '_') + subprocess.check_call( + ['convert', '-size', '100x100', 'label:%s' % text, + '%s/%s' % (OUTPUT_DIR, name)]) + +def is_color_patch(cp): + return cp >= 0x1f3fb and cp <= 0x1f3ff + +def has_color_patch(values): + for v in values: + if is_color_patch(v): + return True + return False + +def regional_to_ascii(cp): + return unichr(ord('A') + cp - 0x1f1e6) + +def is_flag_sequence(values): + if len(values) != 2: + return False + for v in values: + v -= 0x1f1e6 + if v < 0 or v > 25: + return False + return True + +def is_keycap_sequence(values): + return len(values) == 2 and values[1] == 0x20e3 + +def get_keycap_text(values): + return '-%c-' % unichr(values[0]) # convert gags on '[' + +char_map = { + 0x1f468: 'M', + 0x1f469: 'W', + 0x1f466: 'B', + 0x1f467: 'G', + 0x2764: 'H', # heavy black heart, no var sel + 0x1f48b: 'K', # kiss mark + 0x200D: '-', # zwj placeholder + 0xfe0f: '-', # variation selector placeholder + 0x1f441: 'I', # Eye + 0x1f5e8: 'W', # 'witness' (left speech bubble) +} + +def get_combining_text(values): + chars = [] + for v in values: + char = char_map.get(v, None) + if not char: + return None + if char != '-': + chars.append(char) + return ''.join(chars) + + +if not path.isdir(OUTPUT_DIR): + os.makedirs(OUTPUT_DIR) + +with open('sequences.txt', 'r') as f: + for seq in f: + seq = seq.strip() + text = None + values = [int(code, 16) for code in seq.split('_')] + if len(values) == 1: + val = values[0] + text = '%04X' % val # ensure upper case format + elif is_flag_sequence(values): + text = ''.join(regional_to_ascii(cp) for cp in values) + elif has_color_patch(values): + print 'skipping color patch sequence %s' % seq + elif is_keycap_sequence(values): + text = get_keycap_text(values) + else: + text = get_combining_text(values) + if not text: + print 'missing %s' % seq + + if text: + if len(text) > 3: + if len(text) == 4: + hi = text[:2] + lo = text[2:] + else: + hi = text[:-3] + lo = text[-3:] + text = '%s\n%s' % (hi, lo) + generate_image('emoji_u%s.png' % seq, text) diff --git a/third_party/color_emoji/add_glyphs.py b/third_party/color_emoji/add_glyphs.py index a8d986547..26f99b4e9 100644 --- a/third_party/color_emoji/add_glyphs.py +++ b/third_party/color_emoji/add_glyphs.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import glob, os, sys +import collections, glob, os, sys from fontTools import ttx from fontTools.ttLib.tables import otTables from png import PNG @@ -10,11 +10,34 @@ sys.path.append( import add_emoji_gsub +def is_vs(cp): + return cp >= 0xfe00 and cp <= 0xfe0f + +def codes_to_string(codes): + if "_" in codes: + pieces = codes.split ("_") + string = "".join ([unichr (int (code, 16)) for code in pieces]) + else: + try: + string = unichr (int (codes, 16)) + except: + raise ValueError("uh-oh, no unichr for '%s'" % codes) + return string + + +def glyph_sequence(string): + # sequence of names of glyphs that form a ligature + # variation selectors are stripped + return ["u%04X" % ord(char) for char in string if not is_vs(ord(char))] + + def glyph_name(string): + # name of a ligature + # includes variation selectors when present return "_".join (["u%04X" % ord (char) for char in string]) -def add_ligature (font, string): +def add_ligature (font, seq, name): if 'GSUB' not in font: ligature_subst = otTables.LigatureSubst() ligature_subst.ligatures = {} @@ -34,17 +57,27 @@ def add_ligature (font, string): ligatures = lookup.SubTable[0].ligatures lig = otTables.Ligature() - lig.CompCount = len(string) - lig.Component = [glyph_name(ch) for ch in string[1:]] - lig.LigGlyph = glyph_name(string) + lig.CompCount = len(seq) + lig.Component = seq[1:] + lig.LigGlyph = name - first = glyph_name(string[0]) + first = seq[0] try: ligatures[first].append(lig) except KeyError: ligatures[first] = [lig] +# Ligating sequences for emoji that already have a defined codepoint, +# to match the sequences for the related emoji with no codepoint. +# The key is the name of the glyph with the codepoint, the value is the +# name of the sequence in filename form. +EXTRA_SEQUENCES = { + 'u1F46A': '1F468_200D_1F469_200D_1F466', # MWB + 'u1F491': '1F469_200D_2764_FE0F_200D_1F468', # WHM + 'u1F48F': '1F469_200D_2764_FE0F_200D_1F48B_200D_1F468', # WHKM +} + if len (sys.argv) < 4: print >>sys.stderr, """ Usage: @@ -65,23 +98,22 @@ table and the first GSUB lookup (if existing) are modified. in_file = sys.argv[1] out_file = sys.argv[2] -img_prefix = sys.argv[3] +img_prefixen = sys.argv[3:] del sys.argv font = ttx.TTFont() font.importXML (in_file) img_files = {} -glb = "%s*.png" % img_prefix -print "Looking for images matching '%s'." % glb -for img_file in glob.glob (glb): - codes = img_file[len (img_prefix):-4] - if "_" in codes: - pieces = codes.split ("_") - u = "".join ([unichr (int (code, 16)) for code in pieces]) - else: - u = unichr (int (codes, 16)) - img_files[u] = img_file +for img_prefix in img_prefixen: + glb = "%s*.png" % img_prefix + print "Looking for images matching '%s'." % glb + for img_file in glob.glob (glb): + codes = img_file[len (img_prefix):-4] + u = codes_to_string(codes) + if u in img_files: + print 'overwriting %s with %s' % (img_files[u], imag_file) + img_files[u] = img_file if not img_files: raise Exception ("No image files found in '%s'." % glb) @@ -98,20 +130,72 @@ h = font['hmtx'].metrics img_pairs = img_files.items () img_pairs.sort (key=lambda pair: (len (pair[0]), pair[0])) +glyph_names = set() +ligatures = {} + +def add_lig_sequence(ligatures, seq, n): + # Assume sequences with ZWJ are emoji 'ligatures' and rtl order + # is also valid. Internal permutations, though, no. + # We associate a sequence with a filename. We can overwrite the + # sequence with a different filename later. + tseq = tuple(seq) + if tseq in ligatures: + print 'lig sequence %s, replace %s with %s' % ( + tseq, ligatures[tseq], n) + ligatures[tseq] = n + if 'u200D' in seq: + rev_seq = seq[:] + rev_seq.reverse() + trseq = tuple(rev_seq) + # if trseq in ligatures: + # print 'rev lig sequence %s, replace %s with %s' % ( + # trseq, ligatures[trseq], n) + ligatures[trseq] = n + + for (u, filename) in img_pairs: - print "Adding glyph for U+%s" % ",".join (["%04X" % ord (char) for char in u]) + # print "Adding glyph for U+%s" % ",".join (["%04X" % ord (char) for char in u]) n = glyph_name (u) + glyph_names.add(n) + g.append (n) for char in u: - if char not in c: + cp = ord(char) + if cp not in c and not is_vs(cp): name = glyph_name (char) - c[ord (char)] = name + c[cp] = name if len (u) > 1: h[name] = [0, 0] (img_width, img_height) = PNG (filename).get_size () advance = int (round ((float (ascent+descent) * img_width / img_height))) h[n] = [advance, 0] if len (u) > 1: - add_ligature (font, u) + seq = glyph_sequence(u) + add_lig_sequence(ligatures, seq, n) + +for n in EXTRA_SEQUENCES: + if n in glyph_names: + seq = glyph_sequence(codes_to_string(EXTRA_SEQUENCES[n])) + add_lig_sequence(ligatures, seq, n) + else: + print 'extras: no glyph for %s' % n + + +keyed_ligatures = collections.defaultdict(list) +for k, v in ligatures.iteritems(): + first = k[0] + keyed_ligatures[first].append((k, v)) + +for base in sorted(keyed_ligatures): + pairs = keyed_ligatures[base] + # print 'base %s has %d sequences' % (base, len(pairs)) + + # Sort longest first, this ensures longer sequences with common prefixes + # are handled before shorter ones. It would be better to have multiple + # lookups, most likely. + pairs.sort(key = lambda pair: (len(pair[0]), pair[0]), reverse=True) + for seq, name in pairs: + # print seq, name + add_ligature(font, seq, name) font.saveXML (out_file) diff --git a/third_party/color_emoji/emoji_builder.py b/third_party/color_emoji/emoji_builder.py index 5a4e646fe..fa9f4115c 100644 --- a/third_party/color_emoji/emoji_builder.py +++ b/third_party/color_emoji/emoji_builder.py @@ -20,7 +20,8 @@ import sys, struct, StringIO from png import PNG - +import os +from os import path def get_glyph_name_from_gsub (string, font, cmap_dict): ligatures = font['GSUB'].table.LookupList.Lookup[0].SubTable[0].ligatures @@ -83,6 +84,7 @@ class CBDT: write_func = self.image_write_func (image_format) for glyph in glyphs: img_file = glyph_filenames[glyph] + # print 'writing data for glyph %s' % path.basename(img_file) offset = self.tell () write_func (PNG (img_file)) self.glyph_maps.append (GlyphMap (glyph, offset, image_format)) @@ -107,7 +109,11 @@ class CBDT: line_height = (ascent + descent) * y_ppem / float (upem) line_ascent = ascent * y_ppem / float (upem) y_bearing = int (round (line_ascent - .5 * (line_height - height))) + # fudge y_bearing if calculations are a bit off + if y_bearing == 128: + y_bearing = 127 advance = width + # print "small glyph metrics h: %d w: %d" % (height, width) # smallGlyphMetrics # Type Name # BYTE height @@ -115,10 +121,14 @@ class CBDT: # CHAR BearingX # CHAR BearingY # BYTE Advance - self.write (struct.pack ("BBbbB", + try: + self.write (struct.pack ("BBbbB", height, width, x_bearing, y_bearing, advance)) + except Exception as e: + raise ValueError("%s, h: %d w: %d x: %d y: %d %d a:" % ( + e, height, width, x_bearing, y_bearing, advance)) def write_format1 (self, png): @@ -437,8 +447,10 @@ By default they are dropped. eblc.write_header () eblc.start_strikes (len (img_prefixes)) - for img_prefix in img_prefixes: + def is_vs(cp): + return cp >= 0xfe00 and cp <= 0xfe0f + for img_prefix in img_prefixes: print img_files = {} @@ -448,9 +460,14 @@ By default they are dropped. codes = img_file[len (img_prefix):-4] if "_" in codes: pieces = codes.split ("_") - uchars = "".join ([unichr (int (code, 16)) for code in pieces]) + cps = [int(code, 16) for code in pieces] + uchars = "".join ([unichr(cp) for cp in cps if not is_vs(cp)]) else: - uchars = unichr (int (codes, 16)) + cp = int(codes, 16) + if is_vs(cp): + print "ignoring unexpected vs input %04x" % cp + continue + uchars = unichr(cp) img_files[uchars] = img_file if not img_files: raise Exception ("No image files found in '%s'." % glb) @@ -460,14 +477,19 @@ By default they are dropped. advance = width = height = 0 for uchars, img_file in img_files.items (): if len (uchars) == 1: - glyph_name = unicode_cmap.cmap[ord (uchars)] + try: + glyph_name = unicode_cmap.cmap[ord (uchars)] + except: + print "no cmap entry for %x" % ord(uchars) + raise ValueError("%x" % ord(uchars)) else: glyph_name = get_glyph_name_from_gsub (uchars, font, unicode_cmap.cmap) glyph_id = font.getGlyphID (glyph_name) glyph_imgs[glyph_id] = img_file if "verbose" in options: uchars_name = ",".join (["%04X" % ord (char) for char in uchars]) - print "Matched U+%s: id=%d name=%s image=%s" % (uchars_name, glyph_id, glyph_name, img_file) + # print "Matched U+%s: id=%d name=%s image=%s" % ( + # uchars_name, glyph_id, glyph_name, img_file) advance += glyph_metrics[glyph_name][0] w, h = PNG (img_file).get_size () @@ -476,7 +498,7 @@ By default they are dropped. glyphs = sorted (glyph_imgs.keys ()) if not glyphs: - raise Exception ("No common characteres found between font and '%s'." % glb) + raise Exception ("No common characters found between font and '%s'." % glb) print "Embedding images for %d glyphs for this strike." % len (glyphs) advance, width, height = (div (x, len (glyphs)) for x in (advance, width, height))