Upgrade to Python 3

Makefile

@@ -19,6 +19,7 @@ CFLAGS = -std=c99 -Wall -Wextra `pkg-config --cflags --libs cairo`
 LDFLAGS = -lm `pkg-config --libs cairo`
 PNGQUANTDIR := third_party/pngquant
 PNGQUANT := $(PNGQUANTDIR)/pngquant
+PYTHON = python3
 PNGQUANTFLAGS = --speed 1 --skip-if-larger --quality 85-95 --force
 BODY_DIMENSIONS = 136x128
 IMOPS := -size $(BODY_DIMENSIONS) canvas:none -compose copy -gravity center
@@ -30,12 +31,14 @@ ZOPFLIPNG = zopflipng
 OPTIPNG = optipng
 
 EMOJI_BUILDER = third_party/color_emoji/emoji_builder.py
+# flag for emoji builder.  Default to legacy small metrics for the time being.
+SMALL_METRICS := -S
 ADD_GLYPHS = add_glyphs.py
 ADD_GLYPHS_FLAGS = -a emoji_aliases.txt
 PUA_ADDER = map_pua_emoji.py
 VS_ADDER = add_vs_cmap.py # from nototools
 
-EMOJI_SRC_DIR := png/128
+EMOJI_SRC_DIR ?= png/128
 FLAGS_SRC_DIR := third_party/region-flags/png
 
 BUILD_DIR := build
@@ -98,7 +101,7 @@ FLAG_NAMES = $(FLAGS:%=%.png)
 FLAG_FILES = $(addprefix $(FLAGS_DIR)/, $(FLAG_NAMES))
 RESIZED_FLAG_FILES = $(addprefix $(RESIZED_FLAGS_DIR)/, $(FLAG_NAMES))
 
-FLAG_GLYPH_NAMES = $(shell ./flag_glyph_name.py $(FLAGS))
+FLAG_GLYPH_NAMES = $(shell $(PYTHON) flag_glyph_name.py $(FLAGS))
 RENAMED_FLAG_NAMES = $(FLAG_GLYPH_NAMES:%=emoji_%.png)
 RENAMED_FLAG_FILES = $(addprefix $(RENAMED_FLAGS_DIR)/, $(RENAMED_FLAG_NAMES))
 
@@ -219,7 +222,7 @@ endif
 # Run make without -j if this happens.
 
 %.ttx: %.ttx.tmpl $(ADD_GLYPHS) $(ALL_COMPRESSED_FILES)
-	@python $(ADD_GLYPHS) -f "$<" -o "$@" -d "$(COMPRESSED_DIR)" $(ADD_GLYPHS_FLAGS)
+	@$(PYTHON) $(ADD_GLYPHS) -f "$<" -o "$@" -d "$(COMPRESSED_DIR)" $(ADD_GLYPHS_FLAGS)
 
 %.ttf: %.ttx
 	@rm -f "$@"
@@ -227,8 +230,8 @@ endif
 
 $(EMOJI).ttf: $(EMOJI).tmpl.ttf $(EMOJI_BUILDER) $(PUA_ADDER) \
 	$(ALL_COMPRESSED_FILES) | check_vs_adder
-	@python $(EMOJI_BUILDER) -V $< "$@" "$(COMPRESSED_DIR)/emoji_u"
+	@$(PYTHON) $(EMOJI_BUILDER) $(SMALL_METRICS) -V $< "$@" "$(COMPRESSED_DIR)/emoji_u"
-	@python $(PUA_ADDER) "$@" "$@-with-pua"
+	@$(PYTHON) $(PUA_ADDER) "$@" "$@-with-pua"
 	@$(VS_ADDER) -vs 2640 2642 2695 --dstdir '.' -o "$@-with-pua-varsel" "$@-with-pua"
 	@mv "$@-with-pua-varsel" "$@"
 	@rm "$@-with-pua"

add_aliases.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2017 Google Inc. All rights reserved.
 #

add_emoji_gsub.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2014 Google Inc. All rights reserved.
 #

add_glyphs.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Extend a ttx file with additional data.
 
@@ -66,7 +66,7 @@ def collect_seq_to_file(image_dirs, prefix, suffix):
 
 
 def remap_values(seq_to_file, map_fn):
-  return {k: map_fn(v) for k, v in seq_to_file.iteritems()}
+  return {k: map_fn(v) for k, v in seq_to_file.items()}
 
 
 def get_png_file_to_advance_mapper(lineheight):
@@ -228,11 +228,9 @@ def get_rtl_seq(seq):
 
   rev_seq = list(seq)
   rev_seq.reverse()
-  for i in xrange(1, len(rev_seq)):
+  for i in range(len(rev_seq)-1, 0, -1):
     if is_fitzpatrick(rev_seq[i-1]):
-      tmp = rev_seq[i]
+      rev_seq[i-1], rev_seq[i] = rev_seq[i], rev_seq[i-1]
-      rev_seq[i] = rev_seq[i-1]
-      rev_seq[i-1] = tmp
   return tuple(rev_seq)
 
 
@@ -282,7 +280,7 @@ def add_ligature_sequences(font, seqs, aliases):
     return
 
   rtl_seq_to_target_name = {
-      get_rtl_seq(seq): name for seq, name in seq_to_target_name.iteritems()}
+      get_rtl_seq(seq): name for seq, name in seq_to_target_name.items()}
   seq_to_target_name.update(rtl_seq_to_target_name)
   # sequences that don't have rtl variants get mapped to the empty sequence,
   # delete it.
@@ -291,7 +289,7 @@ def add_ligature_sequences(font, seqs, aliases):
 
   # organize by first codepoint in sequence
   keyed_ligatures = collections.defaultdict(list)
-  for t in seq_to_target_name.iteritems():
+  for t in seq_to_target_name.items():
     first_cp = t[0][0]
     keyed_ligatures[first_cp].append(t)
 
@@ -341,7 +339,7 @@ def apply_aliases(seq_dict, aliases):
   source is a key in the dictionary, we can delete it.  This updates the
   dictionary and returns the usable aliases."""
   usable_aliases = {}
-  for k, v in aliases.iteritems():
+  for k, v in aliases.items():
     if v in seq_dict:
       usable_aliases[k] = v
       if k in seq_dict:

add_svg_glyphs.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright 2015 Google, Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

check_emoji_sequences.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Google Inc. All rights reserved.
 #
@@ -26,25 +26,21 @@ import re
 import sys
 
 from nototools import unicode_data
-
+import add_aliases
-DATA_ROOT = path.dirname(path.abspath(__file__))
 
 ZWJ = 0x200d
 EMOJI_VS = 0xfe0f
 
-def _is_regional_indicator(cp):
+END_TAG = 0xe007f
-  return 0x1f1e6 <= cp <= 0x1f1ff
 
+def _make_tag_set():
+  tag_set = set()
+  tag_set |= set(range(0xe0030, 0xe003a))  # 0-9
+  tag_set |= set(range(0xe0061, 0xe007b))  # a-z
+  tag_set.add(END_TAG)
+  return tag_set
 
-def _is_skintone_modifier(cp):
+TAG_SET = _make_tag_set()
-  return 0x1f3fb <= cp <= 0x1f3ff
-
-
-def _seq_string(seq):
-  return '_'.join('%04x' % cp for cp in seq)
-
-def strip_vs(seq):
-  return tuple(cp for cp in seq if cp != EMOJI_VS)
 
 _namedata = None
 
@@ -54,7 +50,7 @@ def seq_name(seq):
   if not _namedata:
     def strip_vs_map(seq_map):
       return {
-          strip_vs(k): v
+          unicode_data.strip_emoji_vs(k): v
           for k, v in seq_map.iteritems()}
     _namedata = [
         strip_vs_map(unicode_data.get_emoji_combining_sequences()),
@@ -70,7 +66,7 @@ def seq_name(seq):
     if seq in data:
       return data[seq]
   if EMOJI_VS in seq:
-    non_vs_seq = strip_vs(seq)
+    non_vs_seq = unicode_data.strip_emoji_vs(seq)
     for data in _namedata:
       if non_vs_seq in data:
         return data[non_vs_seq]
@@ -78,14 +74,29 @@ def seq_name(seq):
   return None
 
 
-def _check_valid_emoji(sorted_seq_to_filepath):
+def _check_no_vs(sorted_seq_to_filepath):
-  """Ensure all emoji are either valid emoji or specific chars."""
+  """Our image data does not use emoji presentation variation selectors."""
+  for seq, fp in sorted_seq_to_filepath.iteritems():
+    if EMOJI_VS in seq:
+      print('check no VS: FE0F in path: %s' % fp)
 
-  valid_cps = set(unicode_data.get_emoji() | unicode_data.proposed_emoji_cps())
+
+def _check_valid_emoji_cps(sorted_seq_to_filepath, unicode_version):
+  """Ensure all cps in these sequences are valid emoji cps or specific cps
+  used in forming emoji sequences.  This is a 'pre-check' that reports
+  this specific problem."""
+
+  valid_cps = set(unicode_data.get_emoji())
+  if unicode_version is None or unicode_version >= unicode_data.PROPOSED_EMOJI_AGE:
+    valid_cps |= unicode_data.proposed_emoji_cps()
+  else:
+    valid_cps = set(
+        cp for cp in valid_cps if unicode_data.age(cp) <= unicode_version)
   valid_cps.add(0x200d)  # ZWJ
   valid_cps.add(0x20e3)  # combining enclosing keycap
   valid_cps.add(0xfe0f)  # variation selector (emoji presentation)
   valid_cps.add(0xfe82b)  # PUA value for unknown flag
+  valid_cps |= TAG_SET  # used in subregion tag sequences
 
   not_emoji = {}
   for seq, fp in sorted_seq_to_filepath.iteritems():
@@ -96,35 +107,43 @@ def _check_valid_emoji(sorted_seq_to_filepath):
         not_emoji[cp].append(fp)
 
   if len(not_emoji):
-    print('%d non-emoji found:' % len(not_emoji), file=sys.stderr)
+    print(
+        'check valid emoji cps: %d non-emoji cp found' % len(not_emoji),
+        file=sys.stderr)
     for cp in sorted(not_emoji):
-      print('%04x (in %s)' % (cp, ', '.join(not_emoji[cp])), file=sys.stderr)
+      fps = not_emoji[cp]
+      print(
+          'check valid emoji cps: %04x (in %d sequences)' % (cp, len(fps)),
+          file=sys.stderr)
 
 
 def _check_zwj(sorted_seq_to_filepath):
-  """Ensure zwj is only between two appropriate emoji."""
+  """Ensure zwj is only between two appropriate emoji.  This is a 'pre-check'
-  ZWJ = 0x200D
+  that reports this specific problem."""
-  EMOJI_PRESENTATION_VS = 0xFE0F
 
   for seq, fp in sorted_seq_to_filepath.iteritems():
     if ZWJ not in seq:
       continue
-    if seq[0] == 0x200d:
+    if seq[0] == ZWJ:
-      print('zwj at head of sequence in %s' % fp, file=sys.stderr)
+      print('check zwj: zwj at head of sequence in %s' % fp, file=sys.stderr)
     if len(seq) == 1:
       continue
-    if seq[-1] == 0x200d:
+    if seq[-1] == ZWJ:
-      print('zwj at end of sequence in %s' % fp, file=sys.stderr)
+      print('check zwj: zwj at end of sequence in %s' % fp, file=sys.stderr)
     for i, cp in enumerate(seq):
       if cp == ZWJ:
         if i > 0:
           pcp = seq[i-1]
-          if pcp != EMOJI_PRESENTATION_VS and not unicode_data.is_emoji(pcp):
+          if pcp != EMOJI_VS and not unicode_data.is_emoji(pcp):
-            print('non-emoji %04x preceeds ZWJ in %s' % (pcp, fp), file=sys.stderr)
+            print(
+                'check zwj: non-emoji %04x preceeds ZWJ in %s' % (pcp, fp),
+                file=sys.stderr)
         if i < len(seq) - 1:
           fcp = seq[i+1]
           if not unicode_data.is_emoji(fcp):
-            print('non-emoji %04x follows ZWJ in %s' % (fcp, fp), file=sys.stderr)
+            print(
+                'check zwj: non-emoji %04x follows ZWJ in %s' % (fcp, fp),
+                file=sys.stderr)
 
 
 def _check_flags(sorted_seq_to_filepath):
@@ -133,15 +152,40 @@ def _check_flags(sorted_seq_to_filepath):
   for seq, fp in sorted_seq_to_filepath.iteritems():
     have_reg = None
     for cp in seq:
-      is_reg = _is_regional_indicator(cp)
+      is_reg = unicode_data.is_regional_indicator(cp)
       if have_reg == None:
         have_reg = is_reg
       elif have_reg != is_reg:
-        print('mix of regional and non-regional in %s' % fp, file=sys.stderr)
+        print(
+            'check flags: mix of regional and non-regional in %s' % fp,
+            file=sys.stderr)
     if have_reg and len(seq) > 2:
       # We provide dummy glyphs for regional indicators, so there are sequences
-      # with single regional indicator symbols.
+      # with single regional indicator symbols, the len check handles this.
-      print('regional indicator sequence length != 2 in %s' % fp, file=sys.stderr)
+      print(
+          'check flags: regional indicator sequence length != 2 in %s' % fp,
+          file=sys.stderr)
+
+def _check_tags(sorted_seq_to_filepath):
+  """Ensure tag sequences (for subregion flags) conform to the spec.  We don't
+  validate against CLDR, just that there's a sequence of 2 or more tags starting
+  and ending with the appropriate codepoints."""
+
+  BLACK_FLAG = 0x1f3f4
+  BLACK_FLAG_SET = set([BLACK_FLAG])
+  for seq, fp in sorted_seq_to_filepath.iteritems():
+    seq_set = set(cp for cp in seq)
+    overlap_set = seq_set & TAG_SET
+    if not overlap_set:
+      continue
+    if seq[0] != BLACK_FLAG:
+      print('check tags: bad start tag in %s' % fp)
+    elif seq[-1] != END_TAG:
+      print('check tags: bad end tag in %s' % fp)
+    elif len(seq) < 4:
+      print('check tags: sequence too short in %s' % fp)
+    elif seq_set - TAG_SET != BLACK_FLAG_SET:
+      print('check tags: non-tag items in %s' % fp)
 
 
 def _check_skintone(sorted_seq_to_filepath):
@@ -151,90 +195,76 @@ def _check_skintone(sorted_seq_to_filepath):
   base_to_modifiers = collections.defaultdict(set)
   for seq, fp in sorted_seq_to_filepath.iteritems():
     for i, cp in enumerate(seq):
-      if _is_skintone_modifier(cp):
+      if unicode_data.is_skintone_modifier(cp):
         if i == 0:
           if len(seq) > 1:
-            print('skin color selector first in sequence %s' % fp, file=sys.stderr)
+            print(
+                'check skintone: skin color selector first in sequence %s' % fp,
+                file=sys.stderr)
           # standalone are ok
           continue
         pcp = seq[i-1]
         if not unicode_data.is_emoji_modifier_base(pcp):
-          print((
+          print(
-              'emoji skintone modifier applied to non-base at %d: %s' % (i, fp)), file=sys.stderr)
+              'check skintone: emoji skintone modifier applied to non-base ' +
-      elif unicode_data.is_emoji_modifier_base(cp):
+              'at %d: %s' % (i, fp), file=sys.stderr)
-        if i < len(seq) - 1 and _is_skintone_modifier(seq[i+1]):
+        else:
-          base_to_modifiers[cp].add(seq[i+1])
+          if pcp not in base_to_modifiers:
-        elif cp not in base_to_modifiers:
+            base_to_modifiers[pcp] = set()
-          base_to_modifiers[cp] = set()
+          base_to_modifiers[pcp].add(cp)
+
   for cp, modifiers in sorted(base_to_modifiers.iteritems()):
     if len(modifiers) != 5:
-      print('emoji base %04x has %d modifiers defined (%s) in %s' % (
+      print(
+          'check skintone: base %04x has %d modifiers defined (%s) in %s' % (
               cp, len(modifiers),
-          ', '.join('%04x' % cp for cp in sorted(modifiers)), fp), file=sys.stderr)
+              ', '.join('%04x' % cp for cp in sorted(modifiers)), fp),
+          file=sys.stderr)
 
 
-def _check_zwj_sequences(seq_to_filepath):
+def _check_zwj_sequences(sorted_seq_to_filepath, unicode_version):
-  """Verify that zwj sequences are valid."""
+  """Verify that zwj sequences are valid for the given unicode version."""
-  zwj_sequence_to_name = unicode_data.get_emoji_zwj_sequences()
+  for seq, fp in sorted_seq_to_filepath.iteritems():
-  # strip emoji variant selectors and add extra mappings
+    if ZWJ not in seq:
-  zwj_sequence_without_vs_to_name_canonical = {}
-  for seq, seq_name in zwj_sequence_to_name.iteritems():
-    if EMOJI_VS in seq:
-      stripped_seq = strip_vs(seq)
-      zwj_sequence_without_vs_to_name_canonical[stripped_seq] = (seq_name, seq)
-
-  zwj_seq_to_filepath = {
-      seq: fp for seq, fp in seq_to_filepath.iteritems()
-      if ZWJ in seq}
-
-  for seq, fp in zwj_seq_to_filepath.iteritems():
-    if seq not in zwj_sequence_to_name:
-      if seq not in zwj_sequence_without_vs_to_name_canonical:
-        print('zwj sequence not defined: %s' % fp, file=sys.stderr)
-      else:
-        _, can = zwj_sequence_without_vs_to_name_canonical[seq]
-        # print >> sys.stderr, 'canonical sequence %s contains vs: %s' % (
-        #     _seq_string(can), fp)
-
-def read_emoji_aliases():
-  result = {}
-
-  with open(path.join(DATA_ROOT, 'emoji_aliases.txt'), 'r') as f:
-    for line in f:
-      ix = line.find('#')
-      if (ix > -1):
-        line = line[:ix]
-      line = line.strip()
-      if not line:
       continue
-      als, trg = (s.strip() for s in line.split(';'))
+    age = unicode_data.get_emoji_sequence_age(seq)
-      als_seq = tuple([int(x, 16) for x in als.split('_')])
+    if age is None or unicode_version is not None and age > unicode_version:
-      try:
+      print('check zwj sequences: undefined sequence %s' % fp)
-        trg_seq = tuple([int(x, 16) for x in trg.split('_')])
-      except:
-        print('cannot process alias %s -> %s' % (als, trg))
-        continue
-      result[als_seq] = trg_seq
-  return result
 
 
-def _check_coverage(seq_to_filepath):
+def _check_no_alias_sources(sorted_seq_to_filepath):
-  age = 9.0
+  """Check that we don't have sequences that we expect to be aliased to
+  some other sequence."""
+  aliases = add_aliases.read_default_emoji_aliases()
+  for seq, fp in sorted_seq_to_filepath.iteritems():
+    if seq in aliases:
+      print('check no alias sources: aliased sequence %s' % fp)
+
+
+def _check_coverage(seq_to_filepath, unicode_version):
+  """Ensure we have all and only the cps and sequences that we need for the
+  font as of this version."""
+
+  age = unicode_version
 
   non_vs_to_canonical = {}
   for k in seq_to_filepath:
     if EMOJI_VS in k:
-      non_vs = strip_vs(k)
+      non_vs = unicode_data.strip_emoji_vs(k)
       non_vs_to_canonical[non_vs] = k
 
-  aliases = read_emoji_aliases()
+  aliases = add_aliases.read_default_emoji_aliases()
   for k, v in sorted(aliases.items()):
     if v not in seq_to_filepath and v not in non_vs_to_canonical:
-      print('alias %s missing target %s' % (_seq_string(k), _seq_string(v)))
+      alias_str = unicode_data.seq_to_string(k)
+      target_str = unicode_data.seq_to_string(v)
+      print('coverage: alias %s missing target %s' % (alias_str, target_str))
       continue
     if k in seq_to_filepath or k in non_vs_to_canonical:
-      print('alias %s already exists as %s (%s)' % (
+      alias_str = unicode_data.seq_to_string(k)
-          _seq_string(k), _seq_string(v), seq_name(v)))
+      target_str = unicode_data.seq_to_string(v)
+      print('coverage: alias %s already exists as %s (%s)' % (
+          alias_str, target_str, seq_name(v)))
       continue
     filename = seq_to_filepath.get(v) or seq_to_filepath[non_vs_to_canonical[v]]
     seq_to_filepath[k] = 'alias:' + filename
@@ -243,13 +273,15 @@ def _check_coverage(seq_to_filepath):
   emoji = sorted(unicode_data.get_emoji(age=age))
   for cp in emoji:
     if tuple([cp]) not in seq_to_filepath:
-      print('missing single %04x (%s)' % (cp, unicode_data.name(cp, '<no name>')))
+      print(
+          'coverage: missing single %04x (%s)' % (
+              cp, unicode_data.name(cp, '<no name>')))
 
   # special characters
   # all but combining enclosing keycap are currently marked as emoji
   for cp in [ord('*'), ord('#'), ord(u'\u20e3')] + range(0x30, 0x3a):
     if cp not in emoji and tuple([cp]) not in seq_to_filepath:
-      print('missing special %04x (%s)' % (cp, unicode_data.name(cp)))
+      print('coverage: missing special %04x (%s)' % (cp, unicode_data.name(cp)))
 
   # combining sequences
   comb_seq_to_name = sorted(
@@ -257,24 +289,26 @@ def _check_coverage(seq_to_filepath):
   for seq, name in comb_seq_to_name:
     if seq not in seq_to_filepath:
       # strip vs and try again
-      non_vs_seq = strip_vs(seq)
+      non_vs_seq = unicode_data.strip_emoji_vs(seq)
       if non_vs_seq not in seq_to_filepath:
-        print('missing combining sequence %s (%s)' % (_seq_string(seq), name))
+        print('coverage: missing combining sequence %s (%s)' %
+              (unicode_data.seq_to_string(seq), name))
 
   # flag sequences
   flag_seq_to_name = sorted(
       unicode_data.get_emoji_flag_sequences(age=age).iteritems())
   for seq, name in flag_seq_to_name:
     if seq not in seq_to_filepath:
-      print('missing flag sequence %s (%s)' % (_seq_string(seq), name))
+      print('coverage: missing flag sequence %s (%s)' %
+            (unicode_data.seq_to_string(seq), name))
 
   # skin tone modifier sequences
   mod_seq_to_name = sorted(
       unicode_data.get_emoji_modifier_sequences(age=age).iteritems())
   for seq, name in mod_seq_to_name:
     if seq not in seq_to_filepath:
-      print('missing modifier sequence %s (%s)' % (
+      print('coverage: missing modifier sequence %s (%s)' % (
-          _seq_string(seq), name))
+          unicode_data.seq_to_string(seq), name))
 
   # zwj sequences
   # some of ours include the emoji presentation variation selector and some
@@ -295,25 +329,30 @@ def _check_coverage(seq_to_filepath):
     else:
       test_seq = seq
     if test_seq not in zwj_seq_without_vs:
-      print('missing (canonical) zwj sequence %s (%s)' % (
+      print('coverage: missing (canonical) zwj sequence %s (%s)' % (
-          _seq_string(seq), name))
+          unicode_data.seq_to_string(seq), name))
 
   # check for 'unknown flag'
-  # this is either emoji_ufe82b or 'unknown_flag', we filter out things that
+  # this is either emoji_ufe82b or 'unknown_flag', but we filter out things that
   # don't start with our prefix so 'unknown_flag' would be excluded by default.
   if tuple([0xfe82b]) not in seq_to_filepath:
-    print('missing unknown flag PUA fe82b')
+    print('coverage: missing unknown flag PUA fe82b')
 
 
-def check_sequence_to_filepath(seq_to_filepath):
+def check_sequence_to_filepath(seq_to_filepath, unicode_version, coverage):
   sorted_seq_to_filepath = collections.OrderedDict(
       sorted(seq_to_filepath.items()))
-  _check_valid_emoji(sorted_seq_to_filepath)
+  _check_no_vs(sorted_seq_to_filepath)
+  _check_valid_emoji_cps(sorted_seq_to_filepath, unicode_version)
   _check_zwj(sorted_seq_to_filepath)
   _check_flags(sorted_seq_to_filepath)
+  _check_tags(sorted_seq_to_filepath)
   _check_skintone(sorted_seq_to_filepath)
-  _check_zwj_sequences(sorted_seq_to_filepath)
+  _check_zwj_sequences(sorted_seq_to_filepath, unicode_version)
-  _check_coverage(sorted_seq_to_filepath)
+  _check_no_alias_sources(sorted_seq_to_filepath)
+  if coverage:
+    _check_coverage(sorted_seq_to_filepath, unicode_version)
+
 
 def create_sequence_to_filepath(name_to_dirpath, prefix, suffix):
   """Check names, and convert name to sequences for names that are ok,
@@ -345,12 +384,15 @@ def create_sequence_to_filepath(name_to_dirpath, prefix, suffix):
   return result
 
 
-def collect_name_to_dirpath(directory, prefix, suffix):
+def collect_name_to_dirpath(directory, prefix, suffix, exclude=None):
   """Return a mapping from filename to path rooted at directory, ignoring files
-  that don't match suffix.  Report when a filename appears in more than one
+  that don't match suffix, and subtrees with names in exclude.  Report when a
-  subdir; the first path found is kept."""
+  filename appears in more than one subdir; the first path found is kept."""
   result = {}
-  for dirname, _, files in os.walk(directory):
+  for dirname, dirs, files in os.walk(directory, topdown=True):
+    if exclude:
+      dirs[:] = [d for d in dirs if d not in exclude]
+
     if directory != '.':
       dirname = path.join(directory, dirname)
     for f in files:
@@ -364,42 +406,57 @@ def collect_name_to_dirpath(directory, prefix, suffix):
   return result
 
 
-def collect_name_to_dirpath_with_override(dirs, prefix, suffix):
+def collect_name_to_dirpath_with_override(dirs, prefix, suffix, exclude=None):
   """Return a mapping from filename to a directory path rooted at a directory
   in dirs, using collect_name_to_filepath.  The last directory is retained. This
   does not report an error if a file appears under more than one root directory,
-  so lets later root directories override earlier ones."""
+  so lets later root directories override earlier ones.  Use 'exclude' to
+  name subdirectories (of any root) whose subtree you wish to skip."""
   result = {}
   for d in dirs:
-    result.update(collect_name_to_dirpath(d, prefix, suffix))
+    result.update(collect_name_to_dirpath(d, prefix, suffix, exclude))
   return result
 
 
-def run_check(dirs, prefix, suffix):
+def run_check(dirs, prefix, suffix, exclude, unicode_version, coverage):
-  print('Checking files with prefix "%s" and suffix "%s" in:\n  %s' % (
+  msg = ''
-      prefix, suffix, '\n  '.join(dirs)))
+  if unicode_version:
+    msg = ' (%3.1f)' % unicode_version
+  print('Checking files with prefix "%s" and suffix "%s"%s in:\n  %s' % (
+      prefix, suffix, msg, '\n  '.join(dirs)))
   name_to_dirpath = collect_name_to_dirpath_with_override(
-      dirs, prefix=prefix, suffix=suffix)
+      dirs, prefix=prefix, suffix=suffix, exclude=exclude)
   print('checking %d names' % len(name_to_dirpath))
   seq_to_filepath = create_sequence_to_filepath(name_to_dirpath, prefix, suffix)
   print('checking %d sequences' % len(seq_to_filepath))
-  check_sequence_to_filepath(seq_to_filepath)
+  check_sequence_to_filepath(seq_to_filepath, unicode_version, coverage)
   print('done.')
 
 
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument(
-      '-d', '--dirs', help='directories containing emoji images',
+      '-d', '--dirs', help='directory roots containing emoji images',
       metavar='dir', nargs='+', required=True)
+  parser.add_argument(
+      '-e', '--exclude', help='names of source subdirs to exclude',
+      metavar='dir', nargs='+')
+  parser.add_argument(
+      '-c', '--coverage', help='test for complete coverage',
+      action='store_true')
   parser.add_argument(
       '-p', '--prefix', help='prefix to match, default "emoji_u"',
       metavar='pfx', default='emoji_u')
   parser.add_argument(
       '-s', '--suffix', help='suffix to match, default ".png"', metavar='sfx',
       default='.png')
+  parser.add_argument(
+      '-u', '--unicode_version', help='limit to this unicode version or before',
+      metavar='version', type=float)
   args = parser.parse_args()
-  run_check(args.dirs, args.prefix, args.suffix)
+  run_check(
+      args.dirs, args.prefix, args.suffix, args.exclude, args.unicode_version,
+      args.coverage)
 
 
 if __name__ == '__main__':

collect_emoji_svg.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright 2015 Google, Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

flag_glyph_name.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2014 Google Inc. All rights reserved.
 #

flag_info.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 #
 # Copyright 2016 Google Inc. All rights reserved.
 #

gen_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2015 Google Inc. All rights reserved.
 #

generate_emoji_html.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Google Inc. All rights reserved.
 #

generate_emoji_name_data.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-#
 #
 # Copyright 2015 Google Inc. All rights reserved.
@@ -390,8 +390,8 @@ def main():
       '-m', '--missing_limit', help='number of missing images before failure '
       '(default 20), use -1 for no limit', metavar='n', default=20)
   parser.add_argument(
-      '--omit_groups', help='names of groups to omit (default "Misc")',
+      '--omit_groups', help='names of groups to omit (default "Misc, Flags")',
-      metavar='name', default=['Misc'], nargs='*')
+      metavar='name', default=['Misc', 'Flags'], nargs='*')
   parser.add_argument(
       '-v', '--verbose', help='print progress information to stdout',
       action='store_true')

generate_emoji_thumbnails.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright 2017 Google Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

generate_test_html.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright 2015 Google, Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

map_pua_emoji.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2014 Google Inc. All rights reserved.
 #
@@ -19,6 +19,7 @@
 __author__ = 'roozbeh@google.com (Roozbeh Pournader)'
 
 import sys
+import itertools
 
 from fontTools import ttLib
 
@@ -53,8 +54,9 @@ def add_pua_cmap(source_file, target_file):
     """Add PUA characters to the cmap of the first font and save as second."""
     font = ttLib.TTFont(source_file)
     cmap = font_data.get_cmap(font)
-    for pua, (ch1, ch2) in (add_emoji_gsub.EMOJI_KEYCAPS.items()
+    for pua, (ch1, ch2) in itertools.chain(
-                            + add_emoji_gsub.EMOJI_FLAGS.items()):
+        add_emoji_gsub.EMOJI_KEYCAPS.items(), add_emoji_gsub.EMOJI_FLAGS.items()
+    ):
         if pua not in cmap:
             glyph_name = get_glyph_name_from_gsub([ch1, ch2], font)
             if glyph_name is not None:

materialize_emoji_images.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Google Inc. All rights reserved.
 #

strip_vs_from_filenames.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2017 Google Inc. All rights reserved.
 #

svg_cleaner.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright 2015 Google, Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -72,18 +72,19 @@ class _Text_Node(object):
 class SvgCleaner(object):
   """Strip out unwanted parts of an svg file, primarily the xml declaration and
   doctype lines, comments, and some attributes of the outermost <svg> element.
-  The id will be replaced when it is inserted into the font.  viewBox causes
+  The id will be replaced when it is inserted into the font. (viewBox causes
   unwanted scaling when used in a font and its effect is difficult to
-  predict. version is unneeded, xml:space is ignored (we're processing spaces
+  predict, but for outside a font we need to keep it sometimes so we keep it).
+  version is unneeded, xml:space is ignored (we're processing spaces
   so a request to maintain them has no effect).  enable-background appears to
   have no effect.  x and y on the outermost svg element have no effect.  We
   keep width and height, and will elsewhere assume these are the dimensions
   used for the character box."""
 
-  def __init__(self):
+  def __init__(self, strip=False):
     self.reader = SvgCleaner._Reader()
     self.cleaner = SvgCleaner._Cleaner()
-    self.writer = SvgCleaner._Writer()
+    self.writer = SvgCleaner._Writer(strip)
 
   class _Reader(object):
     """Loosely based on fonttools's XMLReader.  This generates a tree of nodes,
@@ -130,7 +131,7 @@ class SvgCleaner(object):
 
   class _Cleaner(object):
     def _clean_elem(self, node):
-      viewBox, width, height = None, None, None
+      viewBox, x, y, width, height = None, None, None, None, None
       nattrs = {}
       for k, v in node.attrs.items():
         if node.name == 'svg' and k in [
@@ -153,14 +154,25 @@ class SvgCleaner(object):
         nattrs[k] = v
 
       if node.name == 'svg':
+        if viewBox:
+          x, y, width, height = viewBox.split()
         if not width or not height:
           if not viewBox:
             raise ValueError('no viewBox, width, or height')
-          width, height = viewBox.split()[2:]
         nattrs['width'] = width
         nattrs['height'] = height
+        # keep for svg use outside of font
+        if viewBox and (int(x) != 0 or int(y) != 0):
+          logging.warn('viewbox "%s" x: %s y: %s' % (viewBox, x, y));
+          nattrs['viewBox'] = viewBox
       node.attrs = nattrs
 
+      # if display:none, skip this and its children
+      style = node.attrs.get('style')
+      if (style and 'display:none' in style) or node.attrs.get('display') == 'none':
+          node.contents = []
+          return
+
       # scan contents. remove any empty text nodes, or empty 'g' element nodes.
       # if a 'g' element has no attrs and only one subnode, replace it with the
       # subnode.
@@ -212,6 +224,9 @@ class SvgCleaner(object):
     """For text nodes, replaces sequences of whitespace with a single space.
     For elements, replaces sequences of whitespace in attributes, and
     removes unwanted attributes from <svg> elements."""
+    def __init__(self, strip):
+      logging.warning('writer strip: %s' % strip);
+      self._strip = strip
 
     def _write_node(self, node, lines, indent):
       """Node is a node generated by _Reader, either a TextNode or an
@@ -222,7 +237,7 @@ class SvgCleaner(object):
         if node.text:
           lines.append(node.text)
       else:
-        margin = '  ' * indent
+        margin = '' if self._strip else '  ' * indent
         line = [margin]
         line.append('<%s' % node.name)
         # custom sort attributes of svg, yes this is a hack
@@ -258,7 +273,7 @@ class SvgCleaner(object):
       # the result.
       lines = []
       self._write_node(root, lines, 0)
-      return '\n'.join(lines)
+      return ''.join(lines) if self._strip else '\n'.join(lines)
 
   def tree_from_text(self, svg_text):
     return self.reader.from_text(svg_text)
@@ -276,7 +291,7 @@ class SvgCleaner(object):
     return self.tree_to_text(tree)
 
 
-def clean_svg_files(in_dir, out_dir, match_pat=None, clean=False):
+def clean_svg_files(in_dir, out_dir, match_pat=None, clean=False, strip=False):
   regex = re.compile(match_pat) if match_pat else None
   count = 0
 
@@ -286,7 +301,7 @@ def clean_svg_files(in_dir, out_dir, match_pat=None, clean=False):
 
   out_dir = tool_utils.ensure_dir_exists(out_dir, clean=clean)
 
-  cleaner = SvgCleaner()
+  cleaner = SvgCleaner(strip)
   for file_name in os.listdir(in_dir):
     if regex and not regex.match(file_name):
       continue
@@ -320,6 +335,9 @@ def main():
       metavar='regex', default=None)
   parser.add_argument(
       '-l', '--loglevel', help='log level name/value', default='warning')
+  parser.add_argument(
+      '-w', '--strip_whitespace', help='remove newlines and indentation',
+      action='store_true')
   args = parser.parse_args()
 
   tool_utils.setup_logging(args.loglevel)
@@ -331,7 +349,8 @@ def main():
     logging.info('Writing output to %s', args.out_dir)
 
   clean_svg_files(
-      args.in_dir, args.out_dir, match_pat=args.regex, clean=args.clean)
+      args.in_dir, args.out_dir, match_pat=args.regex, clean=args.clean,
+      strip=args.strip_whitespace)
 
 
 if __name__ == '__main__':

third_party/color_emoji/emoji_builder.py

@@ -19,13 +19,19 @@
 
 
 from __future__ import print_function
-import sys, struct, StringIO
+import sys, struct
 from png import PNG
 import os
 from os import path
 
 from nototools import font_data
 
+
+try:
+	unichr  # py2
+except NameError:
+	unichr = chr  # py3
+
 def get_glyph_name_from_gsub (string, font, cmap_dict):
 	ligatures = font['GSUB'].table.LookupList.Lookup[0].SubTable[0].ligatures
 	first_glyph = cmap_dict[ord (string[0])]
@@ -100,7 +106,7 @@ class CBDT:
 		del self.strike_metrics
 		return glyph_maps
 
-	def write_bigGlyphMetrics (self, width, height):
+	def write_glyphMetrics (self, width, height, big_metrics):
 
 		ascent = self.font_metrics.ascent
 		descent = self.font_metrics.descent
@@ -122,23 +128,30 @@ class CBDT:
 		vert_advance = height
 
 		# print "big glyph metrics h: %d w: %d" % (height, width)
-		# bigGlyphMetrics
+		# smallGlyphMetrics
 		# Type	Name
 		# BYTE	height
 		# BYTE	width
 		# CHAR	horiBearingX
 		# CHAR	horiBearingY
 		# BYTE	horiAdvance
+                # add for bigGlyphMetrics:
 		# CHAR	vertBearingX
 		# CHAR	vertBearingY
 		# BYTE	vertAdvance
 		try:
+		  if big_metrics:
 		        self.write (struct.pack ("BBbbBbbB",
 					 height, width,
 					 x_bearing, y_bearing,
 					 advance,
 					 vert_x_bearing, vert_y_bearing,
 					 vert_advance))
+		  else:
+		        self.write (struct.pack ("BBbbB",
+					 height, width,
+					 x_bearing, y_bearing,
+					 advance))
 		except Exception as e:
 		  raise ValueError("%s, h: %d w: %d x: %d y: %d %d a:" % (
 		      e, height, width, x_bearing, y_bearing, advance))
@@ -172,16 +185,21 @@ class CBDT:
 				self.write (pixel)
 			offset += stride
 
-	png_allowed_chunks =  ["IHDR", "PLTE", "tRNS", "sRGB", "IDAT", "IEND"]
+	png_allowed_chunks =  [b"IHDR", b"PLTE", b"tRNS", b"sRGB", b"IDAT", b"IEND"]
+
+	def write_format17 (self, png):
+                self.write_format17or18(png, False)
 
 	def write_format18 (self, png):
+                self.write_format17or18(png, True)
 
+	def write_format17or18 (self, png, big_metrics):
 		width, height = png.get_size ()
 
 		if 'keep_chunks' not in self.options:
 			png = png.filter_chunks (self.png_allowed_chunks)
 
-		self.write_bigGlyphMetrics (width, height)
+		self.write_glyphMetrics (width, height, big_metrics)
 
 		png_data = png.data ()
 		# ULONG data length
@@ -190,6 +208,7 @@ class CBDT:
 
 	def image_write_func (self, image_format):
 		if image_format == 1: return self.write_format1
+		if image_format == 17: return self.write_format17
 		if image_format == 18: return self.write_format18
 		return None
 
@@ -376,6 +395,7 @@ def main (argv):
 		"-V": "verbose",
 		"-O": "keep_outlines",
 		"-U": "uncompressed",
+                "-S": "small_glyph_metrics",
 		"-C": "keep_chunks",
 	}
 
@@ -388,7 +408,7 @@ def main (argv):
 		print("""
 Usage:
 
-emoji_builder.py [-V] [-O] [-U] [-A] font.ttf out-font.ttf strike-prefix...
+emoji_builder.py [-V] [-O] [-U] [-S] [-A] font.ttf out-font.ttf strike-prefix...
 
 This will search for files that have strike-prefix followed
 by a hex number, and end in ".png".  For example, if strike-prefix
@@ -405,7 +425,10 @@ that the font already supports, and writes the new font out.
 If -V is given, verbose mode is enabled.
 
 If -U is given, uncompressed images are stored (imageFormat=1).
-By default, PNG images are stored (imageFormat=18).
+
+If -S is given, PNG images are stored with small glyph metrics (imageFormat=17).
+
+By default, PNG images are stored with big glyph metrics (imageFormat=18).
 
 If -O is given, the outline tables ('glyf', 'CFF ') and
 related tables are NOT dropped from the font.
@@ -424,7 +447,7 @@ By default they are dropped.
 
 	def add_font_table (font, tag, data):
 		tab = ttLib.tables.DefaultTable.DefaultTable (tag)
-		tab.data = str(data)
+		tab.data = data
 		font[tag] = tab
 
 	def drop_outline_tables (font):
@@ -452,7 +475,8 @@ By default they are dropped.
 	if not unicode_cmap:
 		raise Exception ("Failed to find a Unicode cmap.")
 
-	image_format = 1 if 'uncompressed' in options else 18
+	image_format = 1 if 'uncompressed' in options else (17
+                if 'small_glyph_metrics' in options else 18)
 
 	ebdt = CBDT (font_metrics, options)
 	ebdt.write_header ()
@@ -474,7 +498,7 @@ By default they are dropped.
 			if "_" in codes:
 				pieces = codes.split ("_")
 				cps = [int(code, 16) for code in pieces]
-				uchars = "".join ([unichr(cp) for cp in cps if not is_vs(cp)])
+				uchars = "".join (unichr(cp) for cp in cps if not is_vs(cp))
 			else:
 				cp = int(codes, 16)
 				if is_vs(cp):
@@ -544,7 +568,6 @@ By default they are dropped.
         # remove it earlier, getGlyphID dies.  Need to restructure all of this
         # code.
 	font_data.delete_from_cmap(font, [0xfe82b])
-
 	font.save (out_file)
 	print("Output font '%s' generated." % out_file)
 

third_party/color_emoji/png.py

@@ -17,7 +17,15 @@
 # Google Author(s): Behdad Esfahbod
 #
 
-import struct, StringIO
+import struct
+import sys
+from io import BytesIO
+
+
+try:
+	basestring  # py2
+except NameError:
+	basestring = str  # py3
 
 
 class PNG:
@@ -55,7 +63,8 @@ class PNG:
 		return PNG.signature
 
 	def read_chunk (self):
-		length = struct.unpack (">I", self.f.read (4))[0]
+		buf = self.f.read (4)
+		length = struct.unpack (">I", buf)[0]
 		chunk_type = self.f.read (4)
 		chunk_data = self.f.read (length)
 		if len (chunk_data) != length:
@@ -67,7 +76,7 @@ class PNG:
 
 	def read_IHDR (self):
 		(chunk_type, chunk_data, crc) = self.read_chunk ()
-		if chunk_type != "IHDR":
+		if chunk_type != b"IHDR":
 			raise PNG.BadChunk
 		#  Width:              4 bytes
 		#  Height:             4 bytes
@@ -93,7 +102,7 @@ class PNG:
 
 	def filter_chunks (self, chunks):
 		self.seek (0);
-		out = StringIO.StringIO ()
+		out = BytesIO ()
 		out.write (self.read_signature ())
 		while True:
 			chunk_type, chunk_data, crc = self.read_chunk ()
@@ -102,6 +111,6 @@ class PNG:
 				out.write (chunk_type)
 				out.write (chunk_data)
 				out.write (crc)
-			if chunk_type == "IEND":
+			if chunk_type == b"IEND":
 				break
 		return PNG (out)