Ash
← Commit history

lexer: add Java-like keyword tokens and shared front-end scaffolding

lennoxrose committed on Oct 17, 2026 at 16:46 UTC

995e2a2parent02953c6

13 files changed+652−0

front/.gitignoreadded

+3−0
@@ -0,0 +1,3 @@
1+*.d
2+*.o
3+!.gitkeep

front/Makefileadded

+51−0
@@ -0,0 +1,51 @@
1+CC = gcc
2+.DEFAULT_GOAL := all
3+CFLAGS = -Wall -Wextra -std=c11 -O2 -Isrc
4+
5+# Shared front end for new (Java-like) Ash: lexer, parser, AST, type checker and
6+# lowering, linked by both ashvm and kiln (see MIGRATION.md, decision D12).
7+# Same layout rule as the other subprojects: src/<sub>/C/x.c -> src/<sub>/O/x.o.
8+FRONT_SRC = \
9+ src/jcommon/C/jcommon.c \
10+ src/jlex/C/jlex.c \
11+ src/jparse/C/jparse_decl.c \
12+ src/jparse/C/jparse_expr.c \
13+ src/jparse/C/jparse_stmt.c
14+
15+obj_of = $(patsubst %.c,%.o,$(subst /C/,/O/,$(1)))
16+dep_of = $(patsubst %.c,%.d,$(subst /C/,/D/,$(1)))
17+FRONT_OBJ = $(foreach s,$(FRONT_SRC),$(call obj_of,$(s)))
18+APP_SRC = src/app/C/main.c
19+APP_OBJ = $(foreach s,$(APP_SRC),$(call obj_of,$(s)))
20+ALL_DEP = $(foreach s,$(FRONT_SRC) $(APP_SRC),$(call dep_of,$(s)))
21+
22+define compile_rule
23+$(call obj_of,$(1)): $(1)
24+ @mkdir -p $$(dir $$@) $$(dir $(call dep_of,$(1)))
25+ $$(CC) $$(CFLAGS) -MMD -MP -MF $(call dep_of,$(1)) -MT $$@ -c $$< -o $$@
26+endef
27+$(foreach s,$(FRONT_SRC) $(APP_SRC),$(eval $(call compile_rule,$(s))))
28+
29+LIB = ../bin/libashfront.a
30+TOOL = ../bin/ashfront
31+
32+all: $(LIB) $(TOOL)
33+
34+lib: $(LIB)
35+
36+$(LIB): $(FRONT_OBJ)
37+ mkdir -p ../bin
38+ ar rcs $@ $(FRONT_OBJ)
39+
40+$(TOOL): $(APP_OBJ) $(LIB)
41+ mkdir -p ../bin
42+ $(CC) $(CFLAGS) -o $@ $(APP_OBJ) $(LIB)
43+
44+-include $(ALL_DEP)
45+
46+clean:
47+ find src -name '*.o' -delete
48+ find src -name '*.d' -delete
49+ rm -f $(LIB) $(TOOL)
50+
51+.PHONY: all lib clean

front/src/app/C/main.cadded

+22−0
@@ -0,0 +1,22 @@
1+#include <stdio.h>
2+#include <string.h>
3+
4+#include "jcommon/H/jcommon.h"
5+#include "jparse/H/jparse.h"
6+
7+/* ashfront: developer tool for the new-Ash front end.
8+ * ashfront --parse <file.ash> parse only */
9+int main(int argc, char **argv) {
10+ if (argc < 3) {
11+ fprintf(stderr, "usage: %s --parse <file.ash>\n", argv[0]);
12+ return 2;
13+ }
14+ char *src = j_read_file(argv[2]);
15+ if (!src) {
16+ fprintf(stderr, "error: cannot read %s\n", argv[2]);
17+ return 1;
18+ }
19+ Node *u = jparse_unit(argv[2], src);
20+ printf("parsed %s: %d types\n", argv[2], u->l2.count);
21+ return 0;
22+}

front/src/app/D/.gitkeepadded

+0−0

No content changes. View the file on GitHub

front/src/app/O/.gitkeepadded

+0−0

No content changes. View the file on GitHub

front/src/jcommon/C/jcommon.cadded

+167−0
@@ -0,0 +1,167 @@
1+#include "jcommon/H/jcommon.h"
2+
3+#include <stdarg.h>
4+#include <stdio.h>
5+#include <stdlib.h>
6+#include <string.h>
7+
8+void *j_alloc(size_t n) {
9+ void *p = calloc(1, n ? n : 1);
10+ if (!p) {
11+ fprintf(stderr, "error: out of memory\n");
12+ exit(1);
13+ }
14+ return p;
15+}
16+
17+char *j_strndup(const char *s, size_t n) {
18+ char *p = j_alloc(n + 1);
19+ memcpy(p, s, n);
20+ return p;
21+}
22+
23+char *j_strdup(const char *s) { return j_strndup(s, strlen(s)); }
24+
25+char *j_fmt(const char *fmt, ...) {
26+ va_list ap;
27+ va_start(ap, fmt);
28+ va_list ap2;
29+ va_copy(ap2, ap);
30+ int n = vsnprintf(NULL, 0, fmt, ap);
31+ va_end(ap);
32+ char *p = j_alloc((size_t)n + 1);
33+ vsnprintf(p, (size_t)n + 1, fmt, ap2);
34+ va_end(ap2);
35+ return p;
36+}
37+
38+void j_list_push(JList *l, void *item) {
39+ if (l->count == l->cap) {
40+ l->cap = l->cap ? l->cap * 2 : 4;
41+ l->items = realloc(l->items, sizeof(void *) * (size_t)l->cap);
42+ if (!l->items) {
43+ fprintf(stderr, "error: out of memory\n");
44+ exit(1);
45+ }
46+ }
47+ l->items[l->count++] = item;
48+}
49+
50+static void buf_reserve(JBuf *b, size_t extra) {
51+ if (b->len + extra + 1 > b->cap) {
52+ size_t cap = b->cap ? b->cap : 256;
53+ while (cap < b->len + extra + 1) cap *= 2;
54+ b->data = realloc(b->data, cap);
55+ if (!b->data) {
56+ fprintf(stderr, "error: out of memory\n");
57+ exit(1);
58+ }
59+ b->cap = cap;
60+ }
61+}
62+
63+void j_buf_addn(JBuf *b, const char *s, size_t n) {
64+ buf_reserve(b, n);
65+ memcpy(b->data + b->len, s, n);
66+ b->len += n;
67+ b->data[b->len] = 0;
68+}
69+
70+void j_buf_add(JBuf *b, const char *s) { j_buf_addn(b, s, strlen(s)); }
71+
72+void j_buf_addc(JBuf *b, char c) { j_buf_addn(b, &c, 1); }
73+
74+void j_buf_addf(JBuf *b, const char *fmt, ...) {
75+ va_list ap;
76+ va_start(ap, fmt);
77+ va_list ap2;
78+ va_copy(ap2, ap);
79+ int n = vsnprintf(NULL, 0, fmt, ap);
80+ va_end(ap);
81+ buf_reserve(b, (size_t)n);
82+ vsnprintf(b->data + b->len, (size_t)n + 1, fmt, ap2);
83+ va_end(ap2);
84+ b->len += (size_t)n;
85+}
86+
87+typedef struct {
88+ const char *path;
89+ const char *text;
90+} SourceEntry;
91+
92+static SourceEntry sources[256];
93+static int source_count;
94+static int errors;
95+
96+void j_register_source(const char *path, const char *text) {
97+ if (source_count < 256) {
98+ sources[source_count].path = path;
99+ sources[source_count].text = text;
100+ source_count++;
101+ }
102+}
103+
104+static const char *find_source(const char *path) {
105+ for (int i = 0; i < source_count; i++) {
106+ if (strcmp(sources[i].path, path) == 0) return sources[i].text;
107+ }
108+ return NULL;
109+}
110+
111+static void report(const char *path, int line, int col, const char *fmt, va_list ap) {
112+ fprintf(stderr, "error: ");
113+ vfprintf(stderr, fmt, ap);
114+ fprintf(stderr, "\n --> %s:%d:%d\n", path ? path : "<unknown>", line, col);
115+ const char *text = path ? find_source(path) : NULL;
116+ if (text && line > 0) {
117+ const char *p = text;
118+ for (int l = 1; l < line && *p; p++) {
119+ if (*p == '\n') l++;
120+ }
121+ const char *end = p;
122+ while (*end && *end != '\n') end++;
123+ fprintf(stderr, " |\n%3d| %.*s\n | ", line, (int)(end - p), p);
124+ for (int i = 1; i < col; i++) fputc(' ', stderr);
125+ fprintf(stderr, "^\n");
126+ }
127+}
128+
129+void j_error(const char *path, int line, int col, const char *fmt, ...) {
130+ va_list ap;
131+ va_start(ap, fmt);
132+ report(path, line, col, fmt, ap);
133+ va_end(ap);
134+ exit(1);
135+}
136+
137+void j_error_soft(const char *path, int line, int col, const char *fmt, ...) {
138+ va_list ap;
139+ va_start(ap, fmt);
140+ report(path, line, col, fmt, ap);
141+ va_end(ap);
142+ errors++;
143+}
144+
145+void j_abort_if_errors(void) {
146+ if (errors) {
147+ fprintf(stderr, "%d error%s\n", errors, errors == 1 ? "" : "s");
148+ exit(1);
149+ }
150+}
151+
152+int j_error_count(void) { return errors; }
153+
154+char *j_read_file(const char *path) {
155+ FILE *f = fopen(path, "rb");
156+ if (!f) return NULL;
157+ fseek(f, 0, SEEK_END);
158+ long n = ftell(f);
159+ fseek(f, 0, SEEK_SET);
160+ char *buf = j_alloc((size_t)n + 1);
161+ if (fread(buf, 1, (size_t)n, f) != (size_t)n) {
162+ fclose(f);
163+ return NULL;
164+ }
165+ fclose(f);
166+ return buf;
167+}

front/src/jcommon/D/.gitkeepadded

+0−0

No content changes. View the file on GitHub

front/src/jcommon/H/jcommon.hadded

+47−0
@@ -0,0 +1,47 @@
1+#ifndef JCOMMON_H
2+#define JCOMMON_H
3+
4+#include <stddef.h>
5+
6+/* Shared utilities for the new-Ash front end (lexer, parser, checker, lowering).
7+ * Memory: everything is arena-allocated and never freed; the front end runs once per
8+ * compilation and the process exits afterwards. Errors: j_error reports a Rust-style
9+ * diagnostic and aborts the compilation (exit status 1). */
10+
11+void *j_alloc(size_t n); /* zeroed */
12+char *j_strndup(const char *s, size_t n);
13+char *j_strdup(const char *s);
14+char *j_fmt(const char *fmt, ...);
15+
16+/* Growable pointer list. */
17+typedef struct {
18+ void **items;
19+ int count;
20+ int cap;
21+} JList;
22+
23+void j_list_push(JList *l, void *item);
24+
25+/* Growable string buffer. */
26+typedef struct {
27+ char *data;
28+ size_t len;
29+ size_t cap;
30+} JBuf;
31+
32+void j_buf_add(JBuf *b, const char *s);
33+void j_buf_addn(JBuf *b, const char *s, size_t n);
34+void j_buf_addf(JBuf *b, const char *fmt, ...);
35+void j_buf_addc(JBuf *b, char c);
36+
37+/* Source registry for diagnostics. */
38+void j_register_source(const char *path, const char *text);
39+void j_error(const char *path, int line, int col, const char *fmt, ...);
40+/* Non-fatal: counts the error; call j_abort_if_errors at the end of a phase. */
41+void j_error_soft(const char *path, int line, int col, const char *fmt, ...);
42+void j_abort_if_errors(void);
43+int j_error_count(void);
44+
45+char *j_read_file(const char *path); /* NULL when unreadable */
46+
47+#endif

front/src/jcommon/O/.gitkeepadded

+0−0

No content changes. View the file on GitHub

front/src/jlex/C/jlex.cadded

+306−0
@@ -0,0 +1,306 @@
1+#include "jlex/H/jlex.h"
2+
3+#include <ctype.h>
4+#include <stdlib.h>
5+#include <string.h>
6+
7+#include "jcommon/H/jcommon.h"
8+
9+typedef struct {
10+ const char *word;
11+ JTokKind kind;
12+} Keyword;
13+
14+static const Keyword keywords[] = {
15+ {"package", JT_PACKAGE}, {"import", JT_IMPORT}, {"class", JT_CLASS},
16+ {"interface", JT_INTERFACE}, {"enum", JT_ENUM}, {"extends", JT_EXTENDS},
17+ {"implements", JT_IMPLEMENTS}, {"abstract", JT_ABSTRACT}, {"new", JT_NEW},
18+ {"this", JT_THIS}, {"super", JT_SUPER}, {"public", JT_PUBLIC},
19+ {"private", JT_PRIVATE}, {"protected", JT_PROTECTED}, {"static", JT_STATIC},
20+ {"final", JT_FINAL}, {"const", JT_CONST}, {"native", JT_NATIVE}, {"if", JT_IF},
21+ {"else", JT_ELSE}, {"switch", JT_SWITCH}, {"case", JT_CASE},
22+ {"default", JT_DEFAULT}, {"while", JT_WHILE}, {"for", JT_FOR}, {"do", JT_DO},
23+ {"break", JT_BREAK}, {"continue", JT_CONTINUE}, {"return", JT_RETURN},
24+ {"try", JT_TRY}, {"catch", JT_CATCH}, {"finally", JT_FINALLY},
25+ {"throw", JT_THROW}, {"throws", JT_THROWS}, {"instanceof", JT_INSTANCEOF},
26+ {"void", JT_VOID}, {"true", JT_TRUE}, {"false", JT_FALSE}, {"null", JT_NULL},
27+ {"int", JT_INT}, {"long", JT_LONG}, {"float", JT_FLOAT}, {"double", JT_DOUBLE},
28+ {"bool", JT_BOOL}, {"char", JT_CHAR}, {"byte", JT_BYTE}, {"short", JT_SHORT},
29+};
30+
31+static const char *tok_names[JT_COUNT] = {
32+ [JT_EOF] = "end of file", [JT_IDENT] = "identifier", [JT_INT_LIT] = "integer literal",
33+ [JT_LONG_LIT] = "long literal", [JT_FLOAT_LIT] = "float literal",
34+ [JT_DOUBLE_LIT] = "double literal", [JT_CHAR_LIT] = "char literal",
35+ [JT_STRING_LIT] = "string literal", [JT_LPAREN] = "'('", [JT_RPAREN] = "')'",
36+ [JT_LBRACE] = "'{'", [JT_RBRACE] = "'}'", [JT_LBRACKET] = "'['",
37+ [JT_RBRACKET] = "']'", [JT_SEMI] = "';'", [JT_COMMA] = "','", [JT_DOT] = "'.'",
38+ [JT_COLON] = "':'", [JT_ASSIGN] = "'='", [JT_LT] = "'<'", [JT_GT] = "'>'",
39+ [JT_ARROW] = "'->'", [JT_AT] = "'@'",
40+};
41+
42+const char *jtok_name(JTokKind k) {
43+ for (size_t i = 0; i < sizeof(keywords) / sizeof(keywords[0]); i++) {
44+ if (keywords[i].kind == k) return j_fmt("'%s'", keywords[i].word);
45+ }
46+ return tok_names[k] ? tok_names[k] : "token";
47+}
48+
49+typedef struct {
50+ const char *path;
51+ const char *src;
52+ const char *p;
53+ int line;
54+ const char *line_start;
55+ JList list;
56+} Lexer;
57+
58+static int cur_col(Lexer *lx, const char *at) { return (int)(at - lx->line_start) + 1; }
59+
60+static void push(Lexer *lx, JTokKind kind, const char *text, int line, int col, int adj) {
61+ JToken *t = j_alloc(sizeof(JToken));
62+ t->kind = kind;
63+ t->text = text;
64+ t->line = line;
65+ t->col = col;
66+ t->adj = adj;
67+ j_list_push(&lx->list, t);
68+}
69+
70+/* Reads one (possibly escaped) character from a char/string literal into *out. */
71+static int read_escape(Lexer *lx, int *out) {
72+ char c = *lx->p++;
73+ switch (c) {
74+ case 'n': *out = '\n'; return 1;
75+ case 't': *out = '\t'; return 1;
76+ case 'r': *out = '\r'; return 1;
77+ case '0': *out = 0; return 1;
78+ case '\\': *out = '\\'; return 1;
79+ case '"': *out = '"'; return 1;
80+ case '\'': *out = '\''; return 1;
81+ case 'x': {
82+ int v = 0;
83+ for (int i = 0; i < 2; i++) {
84+ char h = *lx->p;
85+ if (!isxdigit((unsigned char)h)) break;
86+ v = v * 16 + (isdigit((unsigned char)h) ? h - '0' : (tolower(h) - 'a' + 10));
87+ lx->p++;
88+ }
89+ *out = v;
90+ return 1;
91+ }
92+ default:
93+ j_error(lx->path, lx->line, cur_col(lx, lx->p - 1), "unknown escape sequence '\\%c'", c);
94+ }
95+ return 0;
96+}
97+
98+JTokens jlex_tokenize(const char *path, const char *src) {
99+ Lexer lx = {path, src, src, 1, src, {0}};
100+ int prev_end_line = -1;
101+ const char *prev_end = NULL;
102+ for (;;) {
103+ /* whitespace and comments */
104+ for (;;) {
105+ char c = *lx.p;
106+ if (c == '\n') {
107+ lx.p++;
108+ lx.line++;
109+ lx.line_start = lx.p;
110+ } else if (c == ' ' || c == '\t' || c == '\r') {
111+ lx.p++;
112+ } else if (c == '/' && lx.p[1] == '/') {
113+ while (*lx.p && *lx.p != '\n') lx.p++;
114+ } else if (c == '/' && lx.p[1] == '*') {
115+ int sl = lx.line;
116+ const char *sp = lx.p;
117+ int sc = cur_col(&lx, sp);
118+ lx.p += 2;
119+ while (*lx.p && !(lx.p[0] == '*' && lx.p[1] == '/')) {
120+ if (*lx.p == '\n') {
121+ lx.line++;
122+ lx.line_start = lx.p + 1;
123+ }
124+ lx.p++;
125+ }
126+ if (!*lx.p) j_error(path, sl, sc, "unterminated block comment");
127+ lx.p += 2;
128+ } else {
129+ break;
130+ }
131+ }
132+ int line = lx.line;
133+ int col = cur_col(&lx, lx.p);
134+ int adj = prev_end && lx.p == prev_end && prev_end_line == line;
135+ char c = *lx.p;
136+ if (!c) {
137+ push(&lx, JT_EOF, "", line, col, 0);
138+ break;
139+ }
140+ const char *start = lx.p;
141+ if (isalpha((unsigned char)c) || c == '_' || c == '$') {
142+ while (isalnum((unsigned char)*lx.p) || *lx.p == '_' || *lx.p == '$') lx.p++;
143+ size_t n = (size_t)(lx.p - start);
144+ JTokKind kind = JT_IDENT;
145+ for (size_t i = 0; i < sizeof(keywords) / sizeof(keywords[0]); i++) {
146+ if (strlen(keywords[i].word) == n && strncmp(keywords[i].word, start, n) == 0) {
147+ kind = keywords[i].kind;
148+ break;
149+ }
150+ }
151+ push(&lx, kind, j_strndup(start, n), line, col, adj);
152+ } else if (isdigit((unsigned char)c) || (c == '.' && isdigit((unsigned char)lx.p[1]))) {
153+ JTokKind kind = JT_INT_LIT;
154+ if (c == '0' && (lx.p[1] == 'x' || lx.p[1] == 'X')) {
155+ lx.p += 2;
156+ while (isxdigit((unsigned char)*lx.p) || *lx.p == '_') lx.p++;
157+ } else {
158+ while (isdigit((unsigned char)*lx.p) || *lx.p == '_') lx.p++;
159+ if (*lx.p == '.' && isdigit((unsigned char)lx.p[1])) {
160+ kind = JT_DOUBLE_LIT;
161+ lx.p++;
162+ while (isdigit((unsigned char)*lx.p) || *lx.p == '_') lx.p++;
163+ } else if (*lx.p == '.' && !isalpha((unsigned char)lx.p[1]) && lx.p[1] != '.') {
164+ kind = JT_DOUBLE_LIT;
165+ lx.p++;
166+ }
167+ if (*lx.p == 'e' || *lx.p == 'E') {
168+ const char *q = lx.p + 1;
169+ if (*q == '+' || *q == '-') q++;
170+ if (isdigit((unsigned char)*q)) {
171+ kind = JT_DOUBLE_LIT;
172+ lx.p = q;
173+ while (isdigit((unsigned char)*lx.p)) lx.p++;
174+ }
175+ }
176+ }
177+ size_t n = (size_t)(lx.p - start);
178+ char *text = j_alloc(n + 1);
179+ size_t k = 0;
180+ for (size_t i = 0; i < n; i++) {
181+ if (start[i] != '_') text[k++] = start[i];
182+ }
183+ if (*lx.p == 'L' || *lx.p == 'l') {
184+ if (kind != JT_INT_LIT) j_error(path, line, col, "invalid long literal");
185+ kind = JT_LONG_LIT;
186+ lx.p++;
187+ } else if (*lx.p == 'f' || *lx.p == 'F') {
188+ kind = JT_FLOAT_LIT;
189+ lx.p++;
190+ } else if (*lx.p == 'd' || *lx.p == 'D') {
191+ kind = JT_DOUBLE_LIT;
192+ lx.p++;
193+ }
194+ push(&lx, kind, text, line, col, adj);
195+ } else if (c == '"') {
196+ lx.p++;
197+ JBuf b = {0};
198+ while (*lx.p != '"') {
199+ if (!*lx.p || *lx.p == '\n') j_error(path, line, col, "unterminated string literal");
200+ if (*lx.p == '\\') {
201+ lx.p++;
202+ int v;
203+ read_escape(&lx, &v);
204+ j_buf_addc(&b, (char)v);
205+ } else {
206+ j_buf_addc(&b, *lx.p++);
207+ }
208+ }
209+ lx.p++;
210+ push(&lx, JT_STRING_LIT, b.data ? b.data : "", line, col, adj);
211+ } else if (c == '\'') {
212+ lx.p++;
213+ int v;
214+ if (*lx.p == '\\') {
215+ lx.p++;
216+ read_escape(&lx, &v);
217+ } else {
218+ v = (unsigned char)*lx.p++;
219+ }
220+ if (*lx.p != '\'') j_error(path, line, col, "unterminated char literal");
221+ lx.p++;
222+ push(&lx, JT_CHAR_LIT, j_fmt("%d", v), line, col, adj);
223+ } else {
224+ lx.p++;
225+ char n1 = *lx.p;
226+ char n2 = n1 ? lx.p[1] : 0;
227+ JTokKind k = JT_EOF;
228+#define T1(kind) do { k = kind; } while (0)
229+#define T2(kind) do { k = kind; lx.p += 1; } while (0)
230+#define T3(kind) do { k = kind; lx.p += 2; } while (0)
231+ switch (c) {
232+ case '(': T1(JT_LPAREN); break;
233+ case ')': T1(JT_RPAREN); break;
234+ case '{': T1(JT_LBRACE); break;
235+ case '}': T1(JT_RBRACE); break;
236+ case '[': T1(JT_LBRACKET); break;
237+ case ']': T1(JT_RBRACKET); break;
238+ case ';': T1(JT_SEMI); break;
239+ case ',': T1(JT_COMMA); break;
240+ case '@': T1(JT_AT); break;
241+ case '~': T1(JT_TILDE); break;
242+ case '?': T1(JT_QUESTION); break;
243+ case '.':
244+ if (n1 == '.' && n2 == '.') T3(JT_ELLIPSIS); else T1(JT_DOT);
245+ break;
246+ case ':':
247+ if (n1 == ':') T2(JT_COLONCOLON); else T1(JT_COLON);
248+ break;
249+ case '=':
250+ if (n1 == '=') T2(JT_EQ); else T1(JT_ASSIGN);
251+ break;
252+ case '!':
253+ if (n1 == '=') T2(JT_NE); else T1(JT_BANG);
254+ break;
255+ case '+':
256+ if (n1 == '+') T2(JT_INC); else if (n1 == '=') T2(JT_PLUS_ASSIGN); else T1(JT_PLUS);
257+ break;
258+ case '-':
259+ if (n1 == '-') T2(JT_DEC); else if (n1 == '=') T2(JT_MINUS_ASSIGN);
260+ else if (n1 == '>') T2(JT_ARROW); else T1(JT_MINUS);
261+ break;
262+ case '*':
263+ if (n1 == '=') T2(JT_STAR_ASSIGN); else T1(JT_STAR);
264+ break;
265+ case '/':
266+ if (n1 == '=') T2(JT_SLASH_ASSIGN); else T1(JT_SLASH);
267+ break;
268+ case '%':
269+ if (n1 == '=') T2(JT_PERCENT_ASSIGN); else T1(JT_PERCENT);
270+ break;
271+ case '&':
272+ if (n1 == '&') T2(JT_ANDAND); else if (n1 == '=') T2(JT_AMP_ASSIGN); else T1(JT_AMP);
273+ break;
274+ case '|':
275+ if (n1 == '|') T2(JT_OROR); else if (n1 == '=') T2(JT_PIPE_ASSIGN); else T1(JT_PIPE);
276+ break;
277+ case '^':
278+ if (n1 == '=') T2(JT_CARET_ASSIGN); else T1(JT_CARET);
279+ break;
280+ case '<':
281+ if (n1 == '<' && n2 == '=') T3(JT_SHL_ASSIGN);
282+ else if (n1 == '<') T2(JT_SHL);
283+ else if (n1 == '=') T2(JT_LE);
284+ else T1(JT_LT);
285+ break;
286+ case '>':
287+ if (n1 == '=') T2(JT_GE); else T1(JT_GT);
288+ break;
289+ default:
290+ j_error(path, line, col, "unexpected character '%c'", c);
291+ }
292+#undef T1
293+#undef T2
294+#undef T3
295+ push(&lx, k, j_strndup(start, (size_t)(lx.p - start)), line, col, adj);
296+ }
297+ prev_end = lx.p;
298+ prev_end_line = lx.line;
299+ }
300+ JTokens out;
301+ out.path = path;
302+ out.count = lx.list.count;
303+ out.toks = j_alloc(sizeof(JToken) * (size_t)out.count);
304+ for (int i = 0; i < out.count; i++) out.toks[i] = *(JToken *)lx.list.items[i];
305+ return out;
306+}

front/src/jlex/D/.gitkeepadded

+0−0

No content changes. View the file on GitHub

front/src/jlex/H/jlex.hadded

+56−0
@@ -0,0 +1,56 @@
1+#ifndef JLEX_H
2+#define JLEX_H
3+
4+/* Lexer for new (Java-like) Ash. Tokenizes a whole file up front so the parser can
5+ * backtrack. `>` is always a single token (generics); the parser merges adjacent
6+ * `>` tokens into shift operators using the `adj` flag. Comments (//, ///, block)
7+ * are skipped. */
8+
9+typedef enum {
10+ JT_EOF,
11+ JT_IDENT,
12+ JT_INT_LIT, /* 42, 0x1F */
13+ JT_LONG_LIT, /* 42L */
14+ JT_FLOAT_LIT, /* 1.5f */
15+ JT_DOUBLE_LIT, /* 1.5, 1e3 */
16+ JT_CHAR_LIT, /* 'a' (text holds the decoded code point as a number) */
17+ JT_STRING_LIT, /* decoded text */
18+ /* reserved words */
19+ JT_PACKAGE, JT_IMPORT, JT_CLASS, JT_INTERFACE, JT_ENUM, JT_EXTENDS, JT_IMPLEMENTS,
20+ JT_ABSTRACT, JT_NEW, JT_THIS, JT_SUPER, JT_PUBLIC, JT_PRIVATE, JT_PROTECTED,
21+ JT_STATIC, JT_FINAL, JT_CONST, JT_NATIVE, JT_IF, JT_ELSE, JT_SWITCH, JT_CASE,
22+ JT_DEFAULT, JT_WHILE, JT_FOR, JT_DO, JT_BREAK, JT_CONTINUE, JT_RETURN, JT_TRY,
23+ JT_CATCH, JT_FINALLY, JT_THROW, JT_THROWS, JT_INSTANCEOF, JT_VOID, JT_TRUE,
24+ JT_FALSE, JT_NULL, JT_INT, JT_LONG, JT_FLOAT, JT_DOUBLE, JT_BOOL, JT_CHAR,
25+ JT_BYTE, JT_SHORT,
26+ /* punctuation */
27+ JT_LPAREN, JT_RPAREN, JT_LBRACE, JT_RBRACE, JT_LBRACKET, JT_RBRACKET,
28+ JT_SEMI, JT_COMMA, JT_DOT, JT_ELLIPSIS, JT_AT, JT_COLONCOLON, JT_COLON,
29+ JT_QUESTION, JT_ARROW,
30+ /* operators */
31+ JT_ASSIGN, JT_PLUS_ASSIGN, JT_MINUS_ASSIGN, JT_STAR_ASSIGN, JT_SLASH_ASSIGN,
32+ JT_PERCENT_ASSIGN, JT_AMP_ASSIGN, JT_PIPE_ASSIGN, JT_CARET_ASSIGN, JT_SHL_ASSIGN,
33+ JT_EQ, JT_NE, JT_LT, JT_LE, JT_GT, JT_GE, JT_SHL,
34+ JT_PLUS, JT_MINUS, JT_STAR, JT_SLASH, JT_PERCENT, JT_AMP, JT_PIPE, JT_CARET,
35+ JT_TILDE, JT_BANG, JT_ANDAND, JT_OROR, JT_INC, JT_DEC,
36+ JT_COUNT
37+} JTokKind;
38+
39+typedef struct {
40+ JTokKind kind;
41+ const char *text; /* identifier / literal text; decoded for strings */
42+ int line;
43+ int col;
44+ int adj; /* 1 when no whitespace separates this token from the previous one */
45+} JToken;
46+
47+typedef struct {
48+ const char *path;
49+ JToken *toks;
50+ int count;
51+} JTokens;
52+
53+JTokens jlex_tokenize(const char *path, const char *src);
54+const char *jtok_name(JTokKind k);
55+
56+#endif

front/src/jlex/O/.gitkeepadded

+0−0

No content changes. View the file on GitHub

View commit on GitHub← Back to commit history