Skip to content

Commit 3e1f428

Browse files
davidpaquindavidpaquin
authored
Update Zig lexer and example (#1189)
It's been a while since the Zig lexer has been updated, and there have been a number of changes to the language since then. A rough summary of this patch: - Adds new keywords that were not being tracked before, e.g. `opaque` and `anytype` - Removes keywords that are no longer in the language, e.g. `usingnamespace` - Replaces the test file with a larger one that covers more of the language - Follows more best-practice lexer structuring/optimizations, e.g. combining common token sequences when possible
1 parent 31592d5 commit 3e1f428

3 files changed

Lines changed: 16437 additions & 375 deletions

File tree

‎lexers/embedded/zig.xml‎

Lines changed: 111 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,13 @@
88
</config>
99
<rules>
1010
<state name="string">
11-
<rule pattern="\\(x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\&#39;&#34;])">
11+
<rule pattern="\\x[0-9a-fA-F]{2}">
12+
<token type="LiteralStringEscape"/>
13+
</rule>
14+
<rule pattern="\\u\{[0-9a-fA-F]+\}">
15+
<token type="LiteralStringEscape"/>
16+
</rule>
17+
<rule pattern="\\[nrt&#39;&#34;\\]">
1218
<token type="LiteralStringEscape"/>
1319
</rule>
1420
<rule pattern="[^\\&#34;\n]+">
@@ -18,99 +24,164 @@
1824
<token type="LiteralString"/>
1925
<pop depth="1"/>
2026
</rule>
27+
<rule pattern="\\">
28+
<token type="LiteralString"/>
29+
</rule>
30+
</state>
31+
<state name="quoted_ident">
32+
<rule pattern="\\x[0-9a-fA-F]{2}">
33+
<token type="LiteralStringEscape"/>
34+
</rule>
35+
<rule pattern="\\u\{[0-9a-fA-F]+\}">
36+
<token type="LiteralStringEscape"/>
37+
</rule>
38+
<rule pattern="\\[nrt&#39;&#34;\\]">
39+
<token type="LiteralStringEscape"/>
40+
</rule>
41+
<rule pattern="[^\\&#34;\n]+">
42+
<token type="Name"/>
43+
</rule>
44+
<rule pattern="&#34;">
45+
<token type="Name"/>
46+
<pop depth="1"/>
47+
</rule>
48+
<rule pattern="\\">
49+
<token type="Name"/>
50+
</rule>
2151
</state>
2252
<state name="root">
53+
<!-- Whitespace -->
2354
<rule pattern="\n">
2455
<token type="TextWhitespace"/>
2556
</rule>
2657
<rule pattern="\s+">
2758
<token type="TextWhitespace"/>
2859
</rule>
29-
<rule pattern="//.*?\n">
60+
<!-- Comments -->
61+
<rule pattern="//!.*$">
62+
<token type="CommentSpecial"/>
63+
</rule>
64+
<rule pattern="///.*$">
65+
<token type="CommentSpecial"/>
66+
</rule>
67+
<rule pattern="//.*$">
3068
<token type="CommentSingle"/>
3169
</rule>
32-
<rule pattern="(unreachable|continue|errdefer|suspend|return|resume|cancel|break|catch|async|await|defer|asm|try)\b">
70+
<!-- Statement keywords -->
71+
<rule pattern="(break|return|continue|asm|defer|errdefer|unreachable|try|catch|suspend|resume|nosuspend)\b">
3372
<token type="Keyword"/>
3473
</rule>
35-
<rule pattern="(threadlocal|linksection|allowzero|stdcallcc|volatile|comptime|noalias|nakedcc|inline|export|packed|extern|align|const|pub|var)\b">
74+
<!-- Storage/reserved keywords -->
75+
<rule pattern="(const|var|extern|packed|export|pub|noalias|inline|noinline|comptime|volatile|allowzero|align|addrspace|linksection|threadlocal|callconv)\b">
3676
<token type="KeywordReserved"/>
3777
</rule>
38-
<rule pattern="(struct|union|error|enum)\b">
78+
<!-- Structure keywords -->
79+
<rule pattern="(struct|enum|union|error|opaque)\b">
3980
<token type="Keyword"/>
4081
</rule>
82+
<!-- Repeat keywords -->
4183
<rule pattern="(while|for)\b">
4284
<token type="Keyword"/>
4385
</rule>
44-
<rule pattern="(comptime_float|comptime_int|c_longdouble|c_ulonglong|c_longlong|c_voidi8|noreturn|c_ushort|anyerror|promise|c_short|c_ulong|c_uint|c_long|isize|c_int|usize|void|f128|i128|type|bool|u128|u16|f64|f32|u64|i16|f16|i32|u32|i64|u8|i0|u0)\b">
86+
<!-- Type keywords -->
87+
<rule pattern="(bool|void|noreturn|type|anyerror|anyopaque|f16|f32|f64|f80|f128|i8|u8|i16|u16|i32|u32|i64|u64|i128|u128|isize|usize|comptime_int|comptime_float|c_char|c_short|c_ushort|c_int|c_uint|c_long|c_ulong|c_longlong|c_ulonglong|c_longdouble)\b">
4588
<token type="KeywordType"/>
4689
</rule>
47-
<rule pattern="(undefined|false|true|null)\b">
90+
<!-- Constant keywords -->
91+
<rule pattern="(true|false|null|undefined)\b">
4892
<token type="KeywordConstant"/>
4993
</rule>
50-
<rule pattern="(switch|orelse|else|and|if|or)\b">
94+
<!-- Conditional keywords -->
95+
<rule pattern="(if|else|switch|and|or|orelse)\b">
5196
<token type="Keyword"/>
5297
</rule>
53-
<rule pattern="(usingnamespace|test|fn)\b">
98+
<!-- Other keywords -->
99+
<rule pattern="(fn|test|anyframe|anytype)\b">
54100
<token type="Keyword"/>
55101
</rule>
56-
<rule pattern="0x[0-9a-fA-F]+\.[0-9a-fA-F]+([pP][\-+]?[0-9a-fA-F]+)?">
57-
<token type="LiteralNumberFloat"/>
58-
</rule>
59-
<rule pattern="0x[0-9a-fA-F]+\.?[pP][\-+]?[0-9a-fA-F]+">
102+
<!-- Hex float -->
103+
<rule pattern="0x[0-9a-fA-F][0-9a-fA-F_]*(?:\.[0-9a-fA-F][0-9a-fA-F_]*(?:[pP][-+]?[0-9][0-9_]*)?|[pP][-+]?[0-9][0-9_]*)">
60104
<token type="LiteralNumberFloat"/>
61105
</rule>
62-
<rule pattern="[0-9]+\.[0-9]+([eE][-+]?[0-9]+)?">
106+
<!-- Decimal float -->
107+
<rule pattern="[0-9][0-9_]*(?:\.[0-9][0-9_]*(?:[eE][-+]?[0-9][0-9_]*)?|[eE][-+]?[0-9][0-9_]*)">
63108
<token type="LiteralNumberFloat"/>
64109
</rule>
65-
<rule pattern="[0-9]+\.?[eE][-+]?[0-9]+">
66-
<token type="LiteralNumberFloat"/>
67-
</rule>
68-
<rule pattern="0b(?:_?[01])+">
110+
<!-- Binary -->
111+
<rule pattern="0b[01][01_]*">
69112
<token type="LiteralNumberBin"/>
70113
</rule>
71-
<rule pattern="0o(?:_?[0-7])+">
114+
<!-- Octal -->
115+
<rule pattern="0o[0-7][0-7_]*">
72116
<token type="LiteralNumberOct"/>
73117
</rule>
74-
<rule pattern="0x(?:_?[0-9a-fA-F])+">
118+
<!-- Hex integer -->
119+
<rule pattern="0x[0-9a-fA-F][0-9a-fA-F_]*">
75120
<token type="LiteralNumberHex"/>
76121
</rule>
77-
<rule pattern="(?:_?[0-9])+">
122+
<!-- Decimal integer -->
123+
<rule pattern="[0-9][0-9_]*">
78124
<token type="LiteralNumberInteger"/>
79125
</rule>
80-
<rule pattern="\b[A-Za-z_]\w*(?=\s*\()">
81-
<token type="NameFunction"/>
126+
<!-- Quoted identifier -->
127+
<rule pattern="@&#34;">
128+
<token type="Name"/>
129+
<push state="quoted_ident"/>
82130
</rule>
83-
<rule pattern="@[a-zA-Z_]\w*">
131+
<!-- Builtin -->
132+
<rule pattern="@[a-zA-Z_][a-zA-Z0-9_]*">
84133
<token type="NameBuiltin"/>
85134
</rule>
86-
<rule pattern="[a-zA-Z_]\w*">
87-
<token type="Name"/>
88-
</rule>
89-
<rule pattern="\&#39;\\\&#39;\&#39;">
90-
<token type="LiteralStringEscape"/>
135+
<!-- Function call -->
136+
<rule pattern="[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()">
137+
<token type="NameFunction"/>
91138
</rule>
92-
<rule pattern="\&#39;\\(|x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\&#39;&#34;])\&#39;">
93-
<token type="LiteralStringEscape"/>
139+
<!-- Identifier -->
140+
<rule pattern="[a-zA-Z_][a-zA-Z0-9_]*">
141+
<token type="Name"/>
94142
</rule>
95-
<rule pattern="\&#39;[^\\\&#39;]\&#39;">
96-
<token type="LiteralString"/>
143+
<!-- Character literal -->
144+
<rule pattern="&#39;(?:\\x[0-9a-fA-F]{2}|\\u\{[0-9a-fA-F]+\}|\\[nrt&#39;&#34;\\]|[^\\&#39;\n])&#39;">
145+
<token type="LiteralStringChar"/>
97146
</rule>
147+
<!-- Multiline string -->
98148
<rule pattern="\\\\[^\n]*">
99149
<token type="LiteralStringHeredoc"/>
100150
</rule>
101-
<rule pattern="c\\\\[^\n]*">
102-
<token type="LiteralStringHeredoc"/>
103-
</rule>
104-
<rule pattern="c?&#34;">
151+
<!-- String start -->
152+
<rule pattern="&#34;">
105153
<token type="LiteralString"/>
106154
<push state="string"/>
107155
</rule>
108-
<rule pattern="[+%=&gt;&lt;|^!?/\-*&amp;~:]">
156+
<!-- 4-char operators -->
157+
<rule pattern="&lt;&lt;\|=">
158+
<token type="Operator"/>
159+
</rule>
160+
<!-- 3-char operators -->
161+
<rule pattern="(?:\+%=|-%=|\*%=|\+\|=|-\|=|\*\|=|&lt;&lt;=|&gt;&gt;=|&lt;&lt;\|)">
162+
<token type="Operator"/>
163+
</rule>
164+
<!-- 2-char operators -->
165+
<rule pattern="(?:\+\+|\*\*|\|\||&lt;&lt;|&gt;&gt;|\+%|-%|\*%|\+\||-\||\*\||==|!=|&lt;=|&gt;=|\+=|-=|\*=|/=|%=|&amp;=|\|=|\^=|\.\*|\.\?|\.\.)">
166+
<token type="Operator"/>
167+
</rule>
168+
<!-- 1-char operators -->
169+
<rule pattern="[+\-*/%&amp;|^~!&lt;&gt;=]">
109170
<token type="Operator"/>
110171
</rule>
111-
<rule pattern="[{}()\[\],.;]">
172+
<!-- Punctuation -->
173+
<rule pattern="\.\.\.">
174+
<token type="Punctuation"/>
175+
</rule>
176+
<rule pattern="=&gt;">
177+
<token type="Punctuation"/>
178+
</rule>
179+
<rule pattern="-&gt;">
180+
<token type="Punctuation"/>
181+
</rule>
182+
<rule pattern="[{}()\[\],.;:?]">
112183
<token type="Punctuation"/>
113184
</rule>
114185
</state>
115186
</rules>
116-
</lexer>
187+
</lexer>

0 commit comments

Comments
 (0)