Use the new multi-stage unicode table for wcwidth

.gitattributes

@@ -1,4 +1,3 @@
-kitty/wcwidth-std.h linguist-generated=true
 kitty/char-props-data.h linguist-generated=true
 kitty_tests/GraphemeBreakTest.json linguist-generated=true
 kitty/emoji.h linguist-generated=true
@@ -21,7 +20,6 @@ glfw/*.c linguist-vendored=true
 glfw/*.h linguist-vendored=true
 3rdparty/** linguist-vendored=true
 kittens/unicode_input/names.h linguist-generated=true
-tools/wcswidth/std.go linguist-generated=true
 tools/wcswidth/char-props-data.go linguist-generated=true
 tools/unicode_names/names.txt linguist-generated=true
 terminfo/kitty.term* linguist-generated=true

gen/wcwidth.py

@@ -468,73 +468,6 @@ def gofmt(*files: str) -> None:
     subprocess.check_call(['gofmt', '-w', '-s'] + list(files))
 
 
-def gen_wcwidth() -> None:
-    seen: set[int] = set()
-    non_printing = class_maps['Cc'] | class_maps['Cf'] | class_maps['Cs']
-
-    def add(p: Callable[..., None], comment: str, chars_: Union[set[int], frozenset[int]], ret: int, for_go: bool = False) -> None:
-        chars = chars_ - seen
-        seen.update(chars)
-        p(f'\t\t// {comment} ({len(chars)} codepoints)' + ' {{' '{')
-        for spec in get_ranges(list(chars)):
-            write_case(spec, p, for_go)
-            p(f'\t\t\treturn {ret};')
-        p('\t\t// }}}\n')
-
-    def add_all(p: Callable[..., None], for_go: bool = False) -> None:
-        seen.clear()
-        add(p, 'Flags', flag_codepoints, 2, for_go)
-        add(p, 'Marks', marks | {0}, 0, for_go)
-        add(p, 'Non-printing characters', non_printing, -1, for_go)
-        add(p, 'Private use', class_maps['Co'], -3, for_go)
-        add(p, 'Text Presentation', narrow_emoji, 1, for_go)
-        add(p, 'East Asian ambiguous width', ambiguous, -2, for_go)
-        add(p, 'East Asian double width', doublewidth, 2, for_go)
-        add(p, 'Emoji Presentation', wide_emoji, 2, for_go)
-
-        add(p, 'Not assigned in the unicode character database', not_assigned, -4, for_go)
-
-        p('\t\tdefault:\n\t\t\treturn 1;')
-        p('\t}')
-        if for_go:
-            p('\t}')
-        else:
-            p('\treturn 1;\n}')
-
-    with create_header('kitty/wcwidth-std.h') as p, open('tools/wcswidth/std.go', 'w') as gof:
-        gop = partial(print, file=gof)
-        gop('package wcswidth\n\n')
-        gop('func Runewidth(code rune) int {')
-        p('static inline int\nwcwidth_std(int32_t code) {')
-        p('\tif (LIKELY(0x20 <= code && code <= 0x7e)) { return 1; }')
-        p('\tswitch(code) {')
-        gop('\tswitch(code) {')
-        add_all(p)
-        add_all(gop, True)
-
-        p('static inline bool\nis_emoji_presentation_base(uint32_t code) {')
-        gop('func IsEmojiPresentationBase(code rune) bool {')
-        p('\tswitch(code) {')
-        gop('\tswitch(code) {')
-        for spec in get_ranges(list(emoji_presentation_bases)):
-            write_case(spec, p)
-            write_case(spec, gop, for_go=True)
-            p('\t\t\treturn true;')
-            gop('\t\t\treturn true;')
-        p('\t\tdefault: return false;')
-        p('\t}')
-        gop('\t\tdefault:\n\t\t\treturn false')
-        gop('\t}')
-        p('\treturn true;\n}')
-        gop('\n}')
-        uv = unicode_version()
-        p(f'#define UNICODE_MAJOR_VERSION {uv[0]}')
-        p(f'#define UNICODE_MINOR_VERSION {uv[1]}')
-        p(f'#define UNICODE_PATCH_VERSION {uv[2]}')
-        gop('var UnicodeDatabaseVersion [3]int = [3]int{' f'{uv[0]}, {uv[1]}, {uv[2]}' + '}')
-    gofmt(gof.name)
-
-
 def gen_rowcolumn_diacritics() -> None:
     # codes of all row/column diacritics
     codes = []
@@ -814,6 +747,13 @@ func (s CharProps) Width() int {{
 }}''')
         gen_multistage_table(c, gp, t1, t2, shift, mask)
     gofmt(gof.name)
+    with open('kitty/char-props.h', 'r+') as f:
+        raw = f.read()
+        nraw = re.sub(r'\d+/\*=width_shift\*/', f'{width_shift}/*=width_shift*/', raw)
+        if nraw != raw:
+            f.seek(0)
+            f.truncate()
+            f.write(nraw)
 
 
 def main(args: list[str]=sys.argv) -> None:
@@ -823,7 +763,6 @@ def main(args: list[str]=sys.argv) -> None:
     parse_eaw()
     parse_grapheme_segmentation()
     gen_ucd()
-    gen_wcwidth()
     gen_emoji()
     gen_names()
     gen_rowcolumn_diacritics()

kitty/char-props.h

@@ -52,3 +52,4 @@ values:  consonant {extend|linker}* linker {extend|linker}*  */
 CharProps char_props_for(char_type ch);
 void grapheme_segmentation_reset(GraphemeSegmentationState *s);
 bool grapheme_segmentation_step(GraphemeSegmentationState *s, CharProps ch);
+static inline int wcwidth_std(CharProps ch) { return (int)ch.shifted_width - 4/*=width_shift*/; }

kitty/data-types.c

@@ -22,7 +22,6 @@
 #include "cleanup.h"
 #include "safe-wrappers.h"
 #include "control-codes.h"
-#include "wcwidth-std.h"
 #include "wcswidth.h"
 #include "modes.h"
 #include <stddef.h>
@@ -428,7 +427,7 @@ py_shm_unlink(PyObject UNUSED *self, PyObject *args) {
 
 static PyObject*
 wcwidth_wrap(PyObject UNUSED *self, PyObject *chr) {
-    return PyLong_FromLong(wcwidth_std(PyLong_AsLong(chr)));
+    return PyLong_FromLong(wcwidth_std(char_props_for(PyLong_AsLong(chr))));
 }
 
 static PyObject*
@@ -639,7 +638,6 @@ static PyMethodDef module_methods[] = {
     {"expand_ansi_c_escapes", (PyCFunction)expand_ansi_c_escapes, METH_O, ""},
     {"get_docs_ref_map", (PyCFunction)get_docs_ref_map, METH_NOARGS, ""},
     {"wcswidth", (PyCFunction)wcswidth_std, METH_O, ""},
-    {"unicode_database_version", (PyCFunction)unicode_database_version, METH_NOARGS, ""},
     {"open_tty", open_tty, METH_VARARGS, ""},
     {"normal_tty", normal_tty, METH_VARARGS, ""},
     {"raw_tty", raw_tty, METH_VARARGS, ""},

kitty/fast_data_types.pyi

@@ -1699,7 +1699,6 @@ def set_clipboard_data_types(ct: int, mime_types: Tuple[str, ...]) -> None: ...
 def get_clipboard_mime(ct: int, mime: Optional[str], callback: Callable[[bytes], None]) -> None: ...
 def run_with_activation_token(func: Callable[[str], None]) -> bool: ...
 def make_x11_window_a_dock_window(x11_window_id: int, strut: Tuple[int, int, int, int, int, int, int, int, int, int, int, int]) -> None: ...
-def unicode_database_version() -> Tuple[int, int, int]: ...
 def wrapped_kitten_names() -> List[str]: ...
 def expand_ansi_c_escapes(test: str) -> str: ...
 def update_tab_bar_edge_colors(os_window_id: int) -> bool: ...

kitty/freetype_render_ui_text.c

@@ -10,7 +10,7 @@
 #include <hb-ft.h>
 #include "charsets.h"
 #include "unicode-data.h"
-#include "wcwidth-std.h"
+#include "char-props.h"
 #include "wcswidth.h"
 #include FT_BITMAP_H
 #define ELLIPSIS 0x2026
@@ -268,7 +268,7 @@ find_fallback_font_for(RenderCtx *ctx, char_type codep, char_type next_codep) {
     FontConfigFace q;
     bool prefer_color = false;
     char_type string[3] = {codep, next_codep, 0};
-    if (wcswidth_string(string) >= 2 && is_emoji_presentation_base(codep)) prefer_color = true;
+    if (wcswidth_string(string) >= 2 && char_props_for(codep).is_emoji_presentation_base) prefer_color = true;
     if (!fallback_font(codep, main_face_family.name, main_face_family.bold, main_face_family.italic, prefer_color, &q)) return NULL;
     ensure_space_for(&main_face, fallbacks, Face, main_face.count + 1, capacity, 8, true);
     Face *ans = main_face.fallbacks + main_face.count;

kitty/screen.c

@@ -28,7 +28,7 @@
 #include <fcntl.h>
 #include "unicode-data.h"
 #include "modes.h"
-#include "wcwidth-std.h"
+#include "char-props.h"
 #include "wcswidth.h"
 #include <stdalign.h>
 #include "keys.h"
@@ -934,6 +934,11 @@ move_widened_char_past_multiline_chars(Screen *self, CPUCell* cpu_cell, GPUCell
     *cpu_cell = (CPUCell){0}; *gpu_cell = (GPUCell){0};
 }
 
+static bool
+is_emoji_presentation_base(char_type ch) {
+    return char_props_for(ch).is_emoji_presentation_base == 1;
+}
+
 static void
 draw_combining_char(Screen *self, text_loop_state *s, char_type ch) {
     bool has_prev_char = false;
@@ -1100,7 +1105,7 @@ draw_text_loop(Screen *self, const uint32_t *chars, size_t num_chars, text_loop_
                     continue;
                 }
             }
-            char_width = wcwidth_std(ch);
+            char_width = wcwidth_std(char_props_for(ch));
             if (UNLIKELY(char_width < 1)) {
                 if (char_width == 0) continue;
                 char_width = 1;
@@ -4272,7 +4277,7 @@ screen_truncate_point_for_length(PyObject UNUSED *self, PyObject *args) {
                 prev_width = 2;
             } else prev_width = 0;
         } else {
-            int w = wcwidth_std(ch);
+            int w = wcwidth_std(char_props_for(ch));
             switch(w) {
                 case -1:
                 case 0:

kitty/wcswidth.c

@@ -5,7 +5,7 @@
  * Distributed under terms of the GPL3 license.
  */
 
-#include "wcwidth-std.h"
+#include "char-props.h"
 #include "wcswidth.h"
 #include "unicode-data.h"
 
@@ -19,6 +19,11 @@ is_flag_pair(char_type a, char_type b) {
     return is_flag_codepoint(a) && is_flag_codepoint(b);
 }
 
+static inline bool
+is_emoji_presentation_base(char_type ch) {
+    return char_props_for(ch).is_emoji_presentation_base == 1;
+}
+
 int
 wcswidth_step(WCSState *state, const char_type ch) {
     int ans = 0;
@@ -59,7 +64,7 @@ wcswidth_step(WCSState *state, const char_type ch) {
 
                 default: {
                     if (is_flag_codepoint(ch)) state->parser_state = FLAG_PAIR_STARTED;
-                    int w = wcwidth_std(ch);
+                    int w = wcwidth_std(char_props_for(ch));
                     switch(w) {
                         case -1:
                         case 0:
@@ -142,8 +147,3 @@ wcswidth_std(PyObject UNUSED *self, PyObject *str) {
     }
     return PyLong_FromSize_t(ans);
 }
-
-PyObject*
-unicode_database_version(PyObject *self UNUSED, PyObject *args UNUSED) {
-    return Py_BuildValue("iii", UNICODE_MAJOR_VERSION, UNICODE_MINOR_VERSION, UNICODE_PATCH_VERSION);
-}

kitty/wcswidth.h

@@ -20,5 +20,4 @@ typedef struct {
 void initialize_wcs_state(WCSState *state);
 int wcswidth_step(WCSState *state, const char_type ch);
 PyObject * wcswidth_std(PyObject UNUSED *self, PyObject *str);
-PyObject * unicode_database_version(PyObject UNUSED *self, PyObject *str);
 size_t wcswidth_string(const char_type *s);

tools/wcswidth/char-props.go

@@ -130,3 +130,11 @@ func (s *GraphemeSegmentationState) Step(ch CharProps) bool {
 	}
 	return add_to_cell
 }
+
+func Runewidth(code rune) int {
+	return CharPropsFor(code).Width()
+}
+
+func IsEmojiPresentationBase(code rune) bool {
+	return CharPropsFor(code).Is_emoji_presentation_base() == 1
+}