path: root/src/make_unicode_wb_data.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
# make_unicode_wb_data.py
# Copyright (c) 2019-2021  K.Kosako

import sys
import re

MAX_CODE_POINT = 0x10ffff

PR_TOTAL_REG = re.compile("#\s*Total\s+(?:code\s+points|elements):")
PR_LINE_REG  = re.compile("([0-9A-Fa-f]+)(?:..([0-9A-Fa-f]+))?\s*;\s*(\w+)")
PA_LINE_REG  = re.compile("(\w+)\s*;\s*(\w+)")
PVA_LINE_REG = re.compile("(sc|gc)\s*;\s*(\w+)\s*;\s*(\w+)(?:\s*;\s*(\w+))?")
BL_LINE_REG  = re.compile("([0-9A-Fa-f]+)\.\.([0-9A-Fa-f]+)\s*;\s*(.*)")
VERSION_REG  = re.compile("#\s*.*-(\d+)\.(\d+)\.(\d+)\.txt")

VERSION_INFO = [-1, -1, -1]
DIC  = { }
PROPS = []
PropIndex = { }

def check_version_info(s):
  m = VERSION_REG.match(s)
  if m is not None:
    VERSION_INFO[0] = int(m.group(1))
    VERSION_INFO[1] = int(m.group(2))
    VERSION_INFO[2] = int(m.group(3))

def print_ranges(ranges):
  for (start, end) in ranges:
    print "0x%06x, 0x%06x" % (start, end)

def print_prop_and_index(prop, i):
  print "%-35s %3d" % (prop + ',', i)
  PropIndex[prop] = i

def dic_find_by_value(dic, v):
  for key, val in dic.items():
    if val == v:
      return key

  return None


def normalize_ranges(in_ranges, sort=False):
  if sort:
    ranges = sorted(in_ranges)
  else:
    ranges = in_ranges

  r = []
  prev = None
  for (start, end) in ranges:
    if prev >= start - 1:
      (pstart, pend) = r.pop()
      end = max(pend, end)
      start = pstart

    r.append((start, end))
    prev = end

  return r

def inverse_ranges(in_ranges):
  r = []
  prev = 0x000000
  for (start, end) in in_ranges:
    if prev < start:
      r.append((prev, start - 1))

    prev = end + 1

  if prev < MAX_CODE_POINT:
    r.append((prev, MAX_CODE_POINT))

  return r

def add_ranges(r1, r2):
  r = r1 + r2
  return normalize_ranges(r, True)

def sub_one_range(one_range, rs):
  r = []
  (s1, e1) = one_range
  n = len(rs)
  for i in range(0, n):
    (s2, e2) = rs[i]
    if s2 >= s1 and s2 <= e1:
      if s2 > s1:
        r.append((s1, s2 - 1))
      if e2 >= e1:
        return r

      s1 = e2 + 1
    elif s2 < s1 and e2 >= s1:
      if e2 < e1:
        s1 = e2 + 1
      else:
        return r

  r.append((s1, e1))
  return r

def sub_ranges(r1, r2):
  r = []
  for one_range in r1:
    rs = sub_one_range(one_range, r2)
    r.extend(rs)

  return r

def add_ranges_in_dic(dic):
  r = []
  for k, v in dic.items():
    r = r + v

  return normalize_ranges(r, True)

def normalize_ranges_in_dic(dic, sort=False):
  for k, v in dic.items():
    r = normalize_ranges(v, sort)
    dic[k] = r

def merge_dic(to_dic, from_dic):
  to_keys   = to_dic.keys()
  from_keys = from_dic.keys()
  common = list(set(to_keys) & set(from_keys))
  if len(common) != 0:
    print >> sys.stderr, "merge_dic: collision: %s" % sorted(common)

  to_dic.update(from_dic)

def merge_props(to_props, from_props):
  common = list(set(to_props) & set(from_props))
  if len(common) != 0:
    print >> sys.stderr, "merge_props: collision: %s" % sorted(common)

  to_props.extend(from_props)

def add_range_into_dic(dic, name, start, end):
  d = dic.get(name, None)
  if d is None:
    d = [(start, end)]
    dic[name] = d
  else:
    d.append((start, end))

def list_sub(a, b):
  x = set(a) - set(b)
  return list(x)

def parse_properties(path):
  with open(path, 'r') as f:
    dic = { }
    prop = None
    props = []
    for line in f:
      s = line.strip()
      if len(s) == 0:
        continue

      if s[0] == '#':
        if VERSION_INFO[0] < 0:
          check_version_info(s)

      m = PR_LINE_REG.match(s)
      if m:
        prop = m.group(3)
        if m.group(2):
          start = int(m.group(1), 16)
          end   = int(m.group(2), 16)
          add_range_into_dic(dic, prop, start, end)
        else:
          start = int(m.group(1), 16)
          add_range_into_dic(dic, prop, start, start)

      elif PR_TOTAL_REG.match(s) is not None:
        props.append(prop)

  normalize_ranges_in_dic(dic)
  return (dic, props)


### main ###
argv = sys.argv
argc = len(argv)

dic, props = parse_properties('WordBreakProperty.txt')
merge_dic(DIC, dic)
merge_props(PROPS, props)

PROPS = sorted(PROPS)

print '/* unicode_wb_data.c: Generated by make_unicode_wb_data.py. */'
COPYRIGHT = '''
/*-
 * Copyright (c) 2019-2021  K.Kosako
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
'''.strip()

print COPYRIGHT
print ''
if VERSION_INFO[0] < 0:
  raise RuntimeError("Version is not found.")

print "#define WORD_BREAK_PROPERTY_VERSION  %02d%02d%02d" % (VERSION_INFO[0], VERSION_INFO[1], VERSION_INFO[2])
print ''

ranges = []
for prop in PROPS:
  rs = DIC[prop]
  for (start, end) in rs:
    ranges.append((start, end, prop))

ranges = sorted(ranges, key=lambda x: x[0])

prev = -1
for (start, end, prop) in ranges:
  if prev >= start:
    raise ValueError("{2}:{0} - {1} range overlap prev value {3}".format(start, end, prop, prev))


print '/*'
for prop in PROPS:
  print "%s" % prop
print '*/'
print ''

num_ranges = len(ranges)
print "static int WB_RANGE_NUM = %d;" % num_ranges

print 'static WB_RANGE_TYPE WB_RANGES[] = {'
for i, (start, end, prop) in enumerate(ranges):
  if i == num_ranges - 1:
    comma = ''
  else:
    comma = ','

  type_name = 'WB_' + prop
  print " {0x%06x, 0x%06x, %s }%s" % (start, end, type_name, comma)

print '};'

sys.exit(0)