http://git-wip-us.apache.org/repos/asf/lucy-clownfish/blob/23472b18/compiler/modules/CommonMark/src/scanners.h ---------------------------------------------------------------------- diff --git a/compiler/modules/CommonMark/src/scanners.h b/compiler/modules/CommonMark/src/scanners.h new file mode 100644 index 0000000..3269112 --- /dev/null +++ b/compiler/modules/CommonMark/src/scanners.h @@ -0,0 +1,38 @@ +#include "cmark.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int _scan_at(int (*scanner)(const unsigned char *), chunk *c, int offset); +int _scan_autolink_uri(const unsigned char *p); +int _scan_autolink_email(const unsigned char *p); +int _scan_html_tag(const unsigned char *p); +int _scan_html_block_tag(const unsigned char *p); +int _scan_link_url(const unsigned char *p); +int _scan_link_title(const unsigned char *p); +int _scan_spacechars(const unsigned char *p); +int _scan_atx_header_start(const unsigned char *p); +int _scan_setext_header_line(const unsigned char *p); +int _scan_hrule(const unsigned char *p); +int _scan_open_code_fence(const unsigned char *p); +int _scan_close_code_fence(const unsigned char *p); +int _scan_entity(const unsigned char *p); + +#define scan_autolink_uri(c, n) _scan_at(&_scan_autolink_uri, c, n) +#define scan_autolink_email(c, n) _scan_at(&_scan_autolink_email, c, n) +#define scan_html_tag(c, n) _scan_at(&_scan_html_tag, c, n) +#define scan_html_block_tag(c, n) _scan_at(&_scan_html_block_tag, c, n) +#define scan_link_url(c, n) _scan_at(&_scan_link_url, c, n) +#define scan_link_title(c, n) _scan_at(&_scan_link_title, c, n) +#define scan_spacechars(c, n) _scan_at(&_scan_spacechars, c, n) +#define scan_atx_header_start(c, n) _scan_at(&_scan_atx_header_start, c, n) +#define scan_setext_header_line(c, n) _scan_at(&_scan_setext_header_line, c, n) +#define scan_hrule(c, n) _scan_at(&_scan_hrule, c, n) +#define scan_open_code_fence(c, n) _scan_at(&_scan_open_code_fence, c, n) +#define scan_close_code_fence(c, n) _scan_at(&_scan_close_code_fence, c, n) +#define scan_entity(c, n) _scan_at(&_scan_entity, c, n) + +#ifdef __cplusplus +} +#endif
http://git-wip-us.apache.org/repos/asf/lucy-clownfish/blob/23472b18/compiler/modules/CommonMark/src/scanners.re ---------------------------------------------------------------------- diff --git a/compiler/modules/CommonMark/src/scanners.re b/compiler/modules/CommonMark/src/scanners.re new file mode 100644 index 0000000..5e74873 --- /dev/null +++ b/compiler/modules/CommonMark/src/scanners.re @@ -0,0 +1,235 @@ +#include <stdlib.h> +#include "chunk.h" +#include "scanners.h" + +int _scan_at(int (*scanner)(const unsigned char *), chunk *c, int offset) +{ + int res; + unsigned char *ptr = (unsigned char *)c->data; + unsigned char lim = ptr[c->len]; + + ptr[c->len] = '\0'; + res = scanner(ptr + offset); + ptr[c->len] = lim; + + return res; +} + +/*!re2c + re2c:define:YYCTYPE = "unsigned char"; + re2c:define:YYCURSOR = p; + re2c:define:YYMARKER = marker; + re2c:define:YYCTXMARKER = marker; + re2c:yyfill:enable = 0; + + wordchar = [^\x00-\x20]; + + spacechar = [ \t\n]; + + reg_char = [^\\()\x00-\x20]; + + escaped_char = [\\][!"#$%&'()*+,./:;<=>?@[\\\]^_`{|}~-]; + + tagname = [A-Za-z][A-Za-z0-9]*; + + blocktagname = 'article'|'header'|'aside'|'hgroup'|'iframe'|'blockquote'|'hr'|'body'|'li'|'map'|'button'|'object'|'canvas'|'ol'|'caption'|'output'|'col'|'p'|'colgroup'|'pre'|'dd'|'progress'|'div'|'section'|'dl'|'table'|'td'|'dt'|'tbody'|'embed'|'textarea'|'fieldset'|'tfoot'|'figcaption'|'th'|'figure'|'thead'|'footer'|'footer'|'tr'|'form'|'ul'|'h1'|'h2'|'h3'|'h4'|'h5'|'h6'|'video'|'script'|'style'; + + attributename = [a-zA-Z_:][a-zA-Z0-9:._-]*; + + unquotedvalue = [^\"'=<>`\x00]+; + singlequotedvalue = ['][^'\x00]*[']; + doublequotedvalue = [\"][^\"\x00]*[\"]; + + attributevalue = unquotedvalue | singlequotedvalue | doublequotedvalue; + + attributevaluespec = spacechar* [=] spacechar* attributevalue; + + attribute = spacechar+ attributename attributevaluespec?; + + opentag = tagname attribute* spacechar* [/]? [>]; + closetag = [/] tagname spacechar* [>]; + + htmlcomment = "!--" ([^-\x00]+ | [-][^-\x00]+)* "-->"; + + processinginstruction = "?" ([^?>\x00]+ | [?][^>\x00] | [>])* "?>"; + + declaration = "!" [A-Z]+ spacechar+ [^>\x00]* ">"; + + cdata = "![CDATA[" ([^\]\x00]+ | "]" [^\]\x00] | "]]" [^>\x00])* "]]>"; + + htmltag = opentag | closetag | htmlcomment | processinginstruction | + declaration | cdata; + + in_parens_nosp = [(] (reg_char|escaped_char)* [)]; + + in_double_quotes = ["] (escaped_char|[^"\x00])* ["]; + in_single_quotes = ['] (escaped_char|[^'\x00])* [']; + in_parens = [(] (escaped_char|[^)\x00])* [)]; + + scheme = 'coap'|'doi'|'javascript'|'aaa'|'aaas'|'about'|'acap'|'cap'|'cid'|'crid'|'data'|'dav'|'dict'|'dns'|'file'|'ftp'|'geo'|'go'|'gopher'|'h323'|'http'|'https'|'iax'|'icap'|'im'|'imap'|'info'|'ipp'|'iris'|'iris.beep'|'iris.xpc'|'iris.xpcs'|'iris.lwz'|'ldap'|'mailto'|'mid'|'msrp'|'msrps'|'mtqp'|'mupdate'|'news'|'nfs'|'ni'|'nih'|'nntp'|'opaquelocktoken'|'pop'|'pres'|'rtsp'|'service'|'session'|'shttp'|'sieve'|'sip'|'sips'|'sms'|'snmp'|'soap.beep'|'soap.beeps'|'tag'|'tel'|'telnet'|'tftp'|'thismessage'|'tn3270'|'tip'|'tv'|'urn'|'vemmi'|'ws'|'wss'|'xcon'|'xcon-userid'|'xmlrpc.beep'|'xmlrpc.beeps'|'xmpp'|'z39.50r'|'z39.50s'|'adiumxtra'|'afp'|'afs'|'aim'|'apt'|'attachment'|'aw'|'beshare'|'bitcoin'|'bolo'|'callto'|'chrome'|'chrome-extension'|'com-eventbrite-attendee'|'content'|'cvs'|'dlna-playsingle'|'dlna-playcontainer'|'dtn'|'dvb'|'ed2k'|'facetime'|'feed'|'finger'|'fish'|'gg'|'git'|'gizmoproject'|'gtalk'|'hcp'|'icon'|'ipn'|'irc'|'irc6'|'ircs'|'itms'|'jar'|'jms'|'keyparc'|'lastfm'|'lda ps'|'magnet'|'maps'|'market'|'message'|'mms'|'ms-help'|'msnim'|'mumble'|'mvn'|'notes'|'oid'|'palm'|'paparazzi'|'platform'|'proxy'|'psyc'|'query'|'res'|'resource'|'rmi'|'rsync'|'rtmp'|'secondlife'|'sftp'|'sgn'|'skype'|'smb'|'soldat'|'spotify'|'ssh'|'steam'|'svn'|'teamspeak'|'things'|'udp'|'unreal'|'ut2004'|'ventrilo'|'view-source'|'webcal'|'wtai'|'wyciwyg'|'xfire'|'xri'|'ymsgr'; +*/ + +// Try to match URI autolink after first <, returning number of chars matched. +int _scan_autolink_uri(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + scheme [:]([^\x00-\x20<>\\]|escaped_char)*[>] { return (p - start); } + .? { return 0; } +*/ +} + +// Try to match email autolink after first <, returning num of chars matched. +int _scan_autolink_email(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + [a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+ + [@] + [a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])? + ([.][a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)* + [>] { return (p - start); } + .? { return 0; } +*/ +} + +// Try to match an HTML tag after first <, returning num of chars matched. +int _scan_html_tag(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + htmltag { return (p - start); } + .? { return 0; } +*/ +} + +// Try to match an HTML block tag including first <, +// returning num of chars matched. +int _scan_html_block_tag(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + [<] [/] blocktagname (spacechar | [>]) { return (p - start); } + [<] blocktagname (spacechar | [/>]) { return (p - start); } + [<] [!?] { return (p - start); } + .? { return 0; } +*/ +} + +// Try to match a URL in a link or reference, return number of chars matched. +// This may optionally be contained in <..>; otherwise +// whitespace and unbalanced right parentheses aren't allowed. +// Newlines aren't ever allowed. +int _scan_link_url(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + [ \n]* [<] ([^<>\n\\\x00] | escaped_char | [\\])* [>] { return (p - start); } + [ \n]* (reg_char+ | escaped_char | in_parens_nosp)* { return (p - start); } + .? { return 0; } +*/ +} + +// Try to match a link title (in single quotes, in double quotes, or +// in parentheses), returning number of chars matched. Allow one +// level of internal nesting (quotes within quotes). +int _scan_link_title(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + ["] (escaped_char|[^"\x00])* ["] { return (p - start); } + ['] (escaped_char|[^'\x00])* ['] { return (p - start); } + [(] (escaped_char|[^)\x00])* [)] { return (p - start); } + .? { return 0; } +*/ +} + +// Match space characters, including newlines. +int _scan_spacechars(const unsigned char *p) +{ + const unsigned char *start = p; \ +/*!re2c + [ \t\n]* { return (p - start); } + . { return 0; } +*/ +} + +// Match ATX header start. +int _scan_atx_header_start(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + [#]{1,6} ([ ]+|[\n]) { return (p - start); } + .? { return 0; } +*/ +} + +// Match sexext header line. Return 1 for level-1 header, +// 2 for level-2, 0 for no match. +int _scan_setext_header_line(const unsigned char *p) +{ + const unsigned char *marker = NULL; +/*!re2c + [=]+ [ ]* [\n] { return 1; } + [-]+ [ ]* [\n] { return 2; } + .? { return 0; } +*/ +} + +// Scan a horizontal rule line: "...three or more hyphens, asterisks, +// or underscores on a line by themselves. If you wish, you may use +// spaces between the hyphens or asterisks." +int _scan_hrule(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + ([*][ ]*){3,} [ \t]* [\n] { return (p - start); } + ([_][ ]*){3,} [ \t]* [\n] { return (p - start); } + ([-][ ]*){3,} [ \t]* [\n] { return (p - start); } + .? { return 0; } +*/ +} + +// Scan an opening code fence. +int _scan_open_code_fence(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + [`]{3,} / [^`\n\x00]*[\n] { return (p - start); } + [~]{3,} / [^~\n\x00]*[\n] { return (p - start); } + .? { return 0; } +*/ +} + +// Scan a closing code fence with length at least len. +int _scan_close_code_fence(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + ([`]{3,} | [~]{3,}) / spacechar* [\n] { return (p - start); } + .? { return 0; } +*/ +} + +// Scans an entity. +// Returns number of chars matched. +int _scan_entity(const unsigned char *p) +{ + const unsigned char *marker = NULL; + const unsigned char *start = p; +/*!re2c + [&] ([#] ([Xx][A-Fa-f0-9]{1,8}|[0-9]{1,8}) |[A-Za-z][A-Za-z0-9]{1,31} ) [;] + { return (p - start); } + .? { return 0; } +*/ +} http://git-wip-us.apache.org/repos/asf/lucy-clownfish/blob/23472b18/compiler/modules/CommonMark/src/utf8.c ---------------------------------------------------------------------- diff --git a/compiler/modules/CommonMark/src/utf8.c b/compiler/modules/CommonMark/src/utf8.c new file mode 100644 index 0000000..e4ea8e2 --- /dev/null +++ b/compiler/modules/CommonMark/src/utf8.c @@ -0,0 +1,462 @@ +#include <stdlib.h> +#include <stdint.h> +#include <assert.h> + +#include "utf8.h" + +static const int8_t utf8proc_utf8class[256] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 }; + +static void encode_unknown(strbuf *buf) +{ + static const uint8_t repl[] = {239, 191, 189}; + strbuf_put(buf, repl, 3); +} + +static int utf8proc_charlen(const uint8_t *str, int str_len) +{ + int length, i; + + if (!str_len) + return 0; + + length = utf8proc_utf8class[str[0]]; + + if (!length) + return -1; + + if (str_len >= 0 && length > str_len) + return -str_len; + + for (i = 1; i < length; i++) { + if ((str[i] & 0xC0) != 0x80) + return -i; + } + + return length; +} + +// Validate a single UTF-8 character according to RFC 3629. +static int utf8proc_valid(const uint8_t *str, int str_len) +{ + int length = utf8proc_charlen(str, str_len); + + if (length <= 0) + return length; + + switch (length) { + case 1: + if (str[0] == 0x00) { + // ASCII NUL is technically valid but rejected + // for security reasons. + return -length; + } + break; + + case 2: + if (str[0] < 0xC2) { + // Overlong + return -length; + } + break; + + case 3: + if (str[0] == 0xE0) { + if (str[1] < 0xA0) { + // Overlong + return -length; + } + } + else if (str[0] == 0xED) { + if (str[1] >= 0xA0) { + // Surrogate + return -length; + } + } + break; + + case 4: + if (str[0] == 0xF0) { + if (str[1] < 0x90) { + // Overlong + return -length; + } + } + else if (str[0] >= 0xF4) { + if (str[0] > 0xF4 || str[1] >= 0x90) { + // Above 0x10FFFF + return -length; + } + } + break; + } + + return length; +} + +void utf8proc_detab(strbuf *ob, const uint8_t *line, size_t size) +{ + static const uint8_t whitespace[] = " "; + + size_t i = 0, tab = 0; + + while (i < size) { + size_t org = i; + + while (i < size && line[i] != '\t' && line[i] != '\0' + && line[i] < 0x80) { + i++; tab++; + } + + if (i > org) + strbuf_put(ob, line + org, i - org); + + if (i >= size) + break; + + if (line[i] == '\t') { + int numspaces = 4 - (tab % 4); + strbuf_put(ob, whitespace, numspaces); + i += 1; + tab += numspaces; + } else { + int charlen = utf8proc_valid(line + i, size - i); + + if (charlen >= 0) { + strbuf_put(ob, line + i, charlen); + } else { + encode_unknown(ob); + charlen = -charlen; + } + + i += charlen; + tab += 1; + } + } +} + +int utf8proc_iterate(const uint8_t *str, int str_len, int32_t *dst) +{ + int length; + int32_t uc = -1; + + *dst = -1; + length = utf8proc_charlen(str, str_len); + if (length < 0) + return -1; + + switch (length) { + case 1: + uc = str[0]; + break; + case 2: + uc = ((str[0] & 0x1F) << 6) + (str[1] & 0x3F); + if (uc < 0x80) uc = -1; + break; + case 3: + uc = ((str[0] & 0x0F) << 12) + ((str[1] & 0x3F) << 6) + + (str[2] & 0x3F); + if (uc < 0x800 || (uc >= 0xD800 && uc < 0xE000) || + (uc >= 0xFDD0 && uc < 0xFDF0)) uc = -1; + break; + case 4: + uc = ((str[0] & 0x07) << 18) + ((str[1] & 0x3F) << 12) + + ((str[2] & 0x3F) << 6) + (str[3] & 0x3F); + if (uc < 0x10000 || uc >= 0x110000) uc = -1; + break; + } + + if (uc < 0 || ((uc & 0xFFFF) >= 0xFFFE)) + return -1; + + *dst = uc; + return length; +} + +void utf8proc_encode_char(int32_t uc, strbuf *buf) +{ + uint8_t dst[4]; + int len = 0; + + assert(uc >= 0); + + if (uc < 0x80) { + dst[0] = uc; + len = 1; + } else if (uc < 0x800) { + dst[0] = 0xC0 + (uc >> 6); + dst[1] = 0x80 + (uc & 0x3F); + len = 2; + } else if (uc == 0xFFFF) { + dst[0] = 0xFF; + len = 1; + } else if (uc == 0xFFFE) { + dst[0] = 0xFE; + len = 1; + } else if (uc < 0x10000) { + dst[0] = 0xE0 + (uc >> 12); + dst[1] = 0x80 + ((uc >> 6) & 0x3F); + dst[2] = 0x80 + (uc & 0x3F); + len = 3; + } else if (uc < 0x110000) { + dst[0] = 0xF0 + (uc >> 18); + dst[1] = 0x80 + ((uc >> 12) & 0x3F); + dst[2] = 0x80 + ((uc >> 6) & 0x3F); + dst[3] = 0x80 + (uc & 0x3F); + len = 4; + } else { + encode_unknown(buf); + return; + } + + strbuf_put(buf, dst, len); +} + +void utf8proc_case_fold(strbuf *dest, const uint8_t *str, int len) +{ + int32_t c; + +#define bufpush(x) \ + utf8proc_encode_char(x, dest) + + while (len > 0) { + int char_len = utf8proc_iterate(str, len, &c); + + if (char_len >= 0) { +#include "case_fold_switch.inc" + } else { + encode_unknown(dest); + char_len = -char_len; + } + + str += char_len; + len -= char_len; + } +} + +// matches anything in the Zs class, plus LF, CR, TAB, FF. +int utf8proc_is_space(int32_t uc) +{ + return (uc == 9 || + uc == 10 || + uc == 12 || + uc == 13 || + uc == 32 || + uc == 160 || + uc == 5760 || + (uc >= 8192 && uc <= 8202) || + uc == 8239 || + uc == 8287 || + uc == 12288); +} + +// matches anything in the P[cdefios] classes. +int utf8proc_is_punctuation(int32_t uc) +{ + return ((uc >= 33 && uc <= 35) || + (uc >= 37 && uc <= 42) || + (uc >= 44 && uc <= 47) || + uc == 58 || + uc == 59 || + uc == 63 || + uc == 64 || + (uc >= 91 && uc <= 93) || + uc == 95 || + uc == 123 || + uc == 125 || + uc == 161 || + uc == 167 || + uc == 171 || + uc == 182 || + uc == 183 || + uc == 187 || + uc == 191 || + uc == 894 || + uc == 903 || + (uc >= 1370 && uc <= 1375) || + uc == 1417 || + uc == 1418 || + uc == 1470 || + uc == 1472 || + uc == 1475 || + uc == 1478 || + uc == 1523 || + uc == 1524 || + uc == 1545 || + uc == 1546 || + uc == 1548 || + uc == 1549 || + uc == 1563 || + uc == 1566 || + uc == 1567 || + (uc >= 1642 && uc <= 1645) || + uc == 1748 || + (uc >= 1792 && uc <= 1805) || + (uc >= 2039 && uc <= 2041) || + (uc >= 2096 && uc <= 2110) || + uc == 2142 || + uc == 2404 || + uc == 2405 || + uc == 2416 || + uc == 2800 || + uc == 3572 || + uc == 3663 || + uc == 3674 || + uc == 3675 || + (uc >= 3844 && uc <= 3858) || + uc == 3860 || + (uc >= 3898 && uc <= 3901) || + uc == 3973 || + (uc >= 4048 && uc <= 4052) || + uc == 4057 || + uc == 4058 || + (uc >= 4170 && uc <= 4175) || + uc == 4347 || + (uc >= 4960 && uc <= 4968) || + uc == 5120 || + uc == 5741 || + uc == 5742 || + uc == 5787 || + uc == 5788 || + (uc >= 5867 && uc <= 5869) || + uc == 5941 || + uc == 5942 || + (uc >= 6100 && uc <= 6102) || + (uc >= 6104 && uc <= 6106) || + (uc >= 6144 && uc <= 6154) || + uc == 6468 || + uc == 6469 || + uc == 6686 || + uc == 6687 || + (uc >= 6816 && uc <= 6822) || + (uc >= 6824 && uc <= 6829) || + (uc >= 7002 && uc <= 7008) || + (uc >= 7164 && uc <= 7167) || + (uc >= 7227 && uc <= 7231) || + uc == 7294 || + uc == 7295 || + (uc >= 7360 && uc <= 7367) || + uc == 7379 || + (uc >= 8208 && uc <= 8231) || + (uc >= 8240 && uc <= 8259) || + (uc >= 8261 && uc <= 8273) || + (uc >= 8275 && uc <= 8286) || + uc == 8317 || + uc == 8318 || + uc == 8333 || + uc == 8334 || + (uc >= 8968 && uc <= 8971) || + uc == 9001 || + uc == 9002 || + (uc >= 10088 && uc <= 10101) || + uc == 10181 || + uc == 10182 || + (uc >= 10214 && uc <= 10223) || + (uc >= 10627 && uc <= 10648) || + (uc >= 10712 && uc <= 10715) || + uc == 10748 || + uc == 10749 || + (uc >= 11513 && uc <= 11516) || + uc == 11518 || + uc == 11519 || + uc == 11632 || + (uc >= 11776 && uc <= 11822) || + (uc >= 11824 && uc <= 11842) || + (uc >= 12289 && uc <= 12291) || + (uc >= 12296 && uc <= 12305) || + (uc >= 12308 && uc <= 12319) || + uc == 12336 || + uc == 12349 || + uc == 12448 || + uc == 12539 || + uc == 42238 || + uc == 42239 || + (uc >= 42509 && uc <= 42511) || + uc == 42611 || + uc == 42622 || + (uc >= 42738 && uc <= 42743) || + (uc >= 43124 && uc <= 43127) || + uc == 43214 || + uc == 43215 || + (uc >= 43256 && uc <= 43258) || + uc == 43310 || + uc == 43311 || + uc == 43359 || + (uc >= 43457 && uc <= 43469) || + uc == 43486 || + uc == 43487 || + (uc >= 43612 && uc <= 43615) || + uc == 43742 || + uc == 43743 || + uc == 43760 || + uc == 43761 || + uc == 44011 || + uc == 64830 || + uc == 64831 || + (uc >= 65040 && uc <= 65049) || + (uc >= 65072 && uc <= 65106) || + (uc >= 65108 && uc <= 65121) || + uc == 65123 || + uc == 65128 || + uc == 65130 || + uc == 65131 || + (uc >= 65281 && uc <= 65283) || + (uc >= 65285 && uc <= 65290) || + (uc >= 65292 && uc <= 65295) || + uc == 65306 || + uc == 65307 || + uc == 65311 || + uc == 65312 || + (uc >= 65339 && uc <= 65341) || + uc == 65343 || + uc == 65371 || + uc == 65373 || + (uc >= 65375 && uc <= 65381) || + (uc >= 65792 && uc <= 65794) || + uc == 66463 || + uc == 66512 || + uc == 66927 || + uc == 67671 || + uc == 67871 || + uc == 67903 || + (uc >= 68176 && uc <= 68184) || + uc == 68223 || + (uc >= 68336 && uc <= 68342) || + (uc >= 68409 && uc <= 68415) || + (uc >= 68505 && uc <= 68508) || + (uc >= 69703 && uc <= 69709) || + uc == 69819 || + uc == 69820 || + (uc >= 69822 && uc <= 69825) || + (uc >= 69952 && uc <= 69955) || + uc == 70004 || + uc == 70005 || + (uc >= 70085 && uc <= 70088) || + uc == 70093 || + (uc >= 70200 && uc <= 70205) || + uc == 70854 || + (uc >= 71105 && uc <= 71113) || + (uc >= 71233 && uc <= 71235) || + (uc >= 74864 && uc <= 74868) || + uc == 92782 || + uc == 92783 || + uc == 92917 || + (uc >= 92983 && uc <= 92987) || + uc == 92996 || + uc == 113823); +} http://git-wip-us.apache.org/repos/asf/lucy-clownfish/blob/23472b18/compiler/modules/CommonMark/src/utf8.h ---------------------------------------------------------------------- diff --git a/compiler/modules/CommonMark/src/utf8.h b/compiler/modules/CommonMark/src/utf8.h new file mode 100644 index 0000000..7df1573 --- /dev/null +++ b/compiler/modules/CommonMark/src/utf8.h @@ -0,0 +1,22 @@ +#ifndef CMARK_UTF8_H +#define CMARK_UTF8_H + +#include <stdint.h> +#include "buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void utf8proc_case_fold(cmark_strbuf *dest, const uint8_t *str, int len); +void utf8proc_encode_char(int32_t uc, cmark_strbuf *buf); +int utf8proc_iterate(const uint8_t *str, int str_len, int32_t *dst); +void utf8proc_detab(cmark_strbuf *dest, const uint8_t *line, size_t size); +int utf8proc_is_space(int32_t uc); +int utf8proc_is_punctuation(int32_t uc); + +#ifdef __cplusplus +} +#endif + +#endif
