[READ-ONLY] Mirror of https://github.com/probablykasper/cpc. Text calculator with support for units and conversion
cpc.kasper.space
calculator
cli
conversion
converter
library
math
package
units
units-converter
wasm
website
Simplify lexer
+129
-103
+129
-103
src/lexer.rs
+129
-103
src/lexer.rs
···
12
12
use fastnum::D128;
13
13
use fastnum::decimal::Context;
14
14
use std::iter::Peekable;
15
-
use unicode_segmentation::{Graphemes, UnicodeSegmentation};
15
+
use unicode_segmentation::{GraphemeIndices, UnicodeSegmentation};
16
16
17
17
fn is_word_char_str(input: &str) -> bool {
18
18
match input {
···
32
32
)
33
33
}
34
34
35
+
/// For example parse a hyphen with no whitespace before or after it
36
+
fn read_immediate_grapheme(infix: &str, lexer: &mut Lexer) -> bool {
37
+
if let Some((_i, grapheme)) = lexer.graphemes.peek() {
38
+
if *grapheme == infix {
39
+
lexer.graphemes.next();
40
+
return true;
41
+
}
42
+
}
43
+
false
44
+
}
45
+
35
46
/// Read next characters as a word, otherwise return empty string.
36
47
/// Returns an empty string if there's leading whitespace.
37
-
fn read_word_plain(chars: &mut Peekable<Graphemes>) -> String {
48
+
fn read_immediate_word(lexer: &mut Lexer) -> String {
49
+
let graphemes = &mut lexer.graphemes;
50
+
38
51
let mut word = String::new();
39
-
while let Some(next_char) = chars.peek() {
40
-
if is_word_char_str(next_char) {
41
-
word += chars.next().unwrap();
52
+
while let Some((_i, grapheme)) = graphemes.peek() {
53
+
if is_word_char_str(grapheme) {
54
+
word += graphemes.next().unwrap().1;
42
55
} else {
43
56
break;
44
57
}
···
48
61
49
62
/// Read next as a word, otherwise return empty string.
50
63
/// Leading whitespace is ignored. A trailing digit may be included.
51
-
fn read_word(first_c: &str, lexer: &mut Lexer) -> String {
52
-
let chars = &mut lexer.chars;
53
-
let mut word = first_c.trim().to_owned();
54
-
if word.is_empty() {
55
-
// skip whitespace
56
-
while let Some(current_char) = chars.peek() {
57
-
if current_char.trim().is_empty() {
58
-
chars.next();
59
-
} else {
60
-
break;
61
-
}
64
+
fn read_word(lexer: &mut Lexer) -> String {
65
+
let graphemes = &mut lexer.graphemes;
66
+
// skip whitespace
67
+
while let Some((_i, grapheme)) = graphemes.peek() {
68
+
if grapheme.trim_start().is_empty() {
69
+
graphemes.next();
70
+
} else {
71
+
break;
62
72
}
63
73
}
64
-
while let Some(next_char) = chars.peek() {
65
-
if is_word_char_str(next_char) {
66
-
word += chars.next().unwrap();
74
+
let mut word = "".to_string();
75
+
while let Some((_i, grapheme)) = graphemes.peek() {
76
+
if is_word_char_str(grapheme) {
77
+
word += graphemes.next().unwrap().1;
67
78
} else {
68
79
break;
69
80
}
70
81
}
71
82
if !word.is_empty() {
72
-
match *chars.peek().unwrap_or(&"") {
83
+
match *graphemes.peek().map(|(_i, g)| g).unwrap_or(&"") {
73
84
"2" | "²" => {
74
85
word += "2";
75
-
chars.next();
86
+
graphemes.next();
76
87
}
77
88
"3" | "³" => {
78
89
word += "3";
79
-
chars.next();
90
+
graphemes.next();
80
91
}
81
92
_ => {}
82
93
}
···
84
95
word
85
96
}
86
97
87
-
fn parse_token(c: &str, lexer: &mut Lexer) -> Result<(), String> {
88
-
let tokens = &mut lexer.tokens;
89
-
match c {
90
-
value if value.trim().is_empty() => {}
91
-
value if is_word_char_str(value) => {
92
-
parse_word(read_word(c, lexer).as_str(), lexer)?;
98
+
fn lex_token(lexer: &mut Lexer) -> Result<(), String> {
99
+
let (start_i, first_grapheme) = match lexer.graphemes.peek() {
100
+
Some(c) => *c,
101
+
None => return Ok(()),
102
+
};
103
+
let token = match first_grapheme {
104
+
grapheme if grapheme.trim_start().is_empty() => {
105
+
lexer.graphemes.next();
106
+
return Ok(());
107
+
}
108
+
grapheme if is_word_char_str(grapheme) => {
109
+
lex_word(read_word(lexer).as_str(), lexer)?;
110
+
return Ok(());
93
111
}
94
-
value if is_numeric_str(value) => {
95
-
let mut number_string = value.to_owned();
96
-
while let Some(number_char) = lexer.chars.peek() {
97
-
if is_numeric_str(number_char) {
98
-
number_string += number_char;
99
-
lexer.chars.next();
112
+
grapheme if is_numeric_str(grapheme) => {
113
+
let mut end_i = start_i + grapheme.len();
114
+
lexer.graphemes.next();
115
+
while let Some((_, grapheme)) = lexer.graphemes.peek() {
116
+
if is_numeric_str(grapheme) {
117
+
end_i += grapheme.len();
118
+
lexer.graphemes.next();
100
119
} else {
101
120
break;
102
121
}
103
122
}
104
-
match D128::from_str(&number_string, Context::default()) {
105
-
Ok(number) => {
106
-
tokens.push(Token::Number(number));
107
-
}
123
+
let number_string = &lexer.input[start_i..end_i];
124
+
let token = match D128::from_str(&number_string, Context::default()) {
125
+
Ok(number) => Token::Number(number),
108
126
Err(_e) => {
109
127
return Err(format!("Error lexing d128 number: {}", number_string));
110
128
}
111
129
};
130
+
lexer.tokens.push(token);
131
+
return Ok(());
112
132
}
113
-
"+" => tokens.push(Token::Operator(Plus)),
114
-
"-" => tokens.push(Token::Operator(Minus)),
115
-
"*" => tokens.push(Token::Operator(Multiply)),
116
-
"/" | "÷" => tokens.push(Token::Operator(Divide)),
117
-
"%" => tokens.push(Token::LexerKeyword(PercentChar)),
118
-
"^" => tokens.push(Token::Operator(Caret)),
119
-
"!" => tokens.push(Token::UnaryOperator(Factorial)),
133
+
"+" => Token::Operator(Plus),
134
+
"-" => Token::Operator(Minus),
135
+
"*" => Token::Operator(Multiply),
136
+
"/" | "÷" => Token::Operator(Divide),
137
+
"%" => Token::LexerKeyword(PercentChar),
138
+
"^" => Token::Operator(Caret),
139
+
"!" => Token::UnaryOperator(Factorial),
120
140
"(" => {
121
141
lexer.left_paren_count += 1;
122
-
tokens.push(Token::Operator(LeftParen));
142
+
Token::Operator(LeftParen)
123
143
}
124
144
")" => {
125
145
lexer.right_paren_count += 1;
126
-
tokens.push(Token::Operator(RightParen));
146
+
Token::Operator(RightParen)
127
147
}
128
-
"π" => tokens.push(Token::Constant(Pi)),
129
-
"'" => tokens.push(Token::Unit(Foot)),
130
-
"\"" | "“" | "”" | "″" => tokens.push(Token::LexerKeyword(DoubleQuotes)),
131
-
_ => {
132
-
return Err(format!("Invalid character: {}", c));
148
+
"π" => Token::Constant(Pi),
149
+
"'" => Token::Unit(Foot),
150
+
"\"" | "“" | "”" | "″" => Token::LexerKeyword(DoubleQuotes),
151
+
grapheme => {
152
+
return Err(format!("Invalid character: {}", grapheme));
133
153
}
134
-
}
154
+
};
155
+
lexer.graphemes.next();
156
+
lexer.tokens.push(token);
135
157
Ok(())
136
158
}
137
159
138
-
fn parse_word_if_non_empty(word: &str, lexer: &mut Lexer) -> Result<(), String> {
160
+
fn lex_word_if_non_empty(word: &str, lexer: &mut Lexer) -> Result<(), String> {
139
161
match word {
140
162
"" => Ok(()),
141
-
_ => parse_word(word, lexer),
163
+
_ => lex_word(word, lexer),
142
164
}
143
165
}
144
166
145
-
fn parse_word(word: &str, lexer: &mut Lexer) -> Result<(), String> {
167
+
fn lex_word(word: &str, lexer: &mut Lexer) -> Result<(), String> {
146
168
let token = match word {
147
169
"to" => Token::TextOperator(To),
148
170
"of" => Token::TextOperator(Of),
···
178
200
"plus" => Token::Operator(Plus),
179
201
"minus" => Token::Operator(Minus),
180
202
"times" => Token::Operator(Multiply),
181
-
"multiplied" => match read_word("", lexer).as_str() {
203
+
"multiplied" => match read_word(lexer).as_str() {
182
204
"by" => Token::Operator(Multiply),
183
205
string => return Err(format!("Invalid string: {}", string)),
184
206
},
185
-
"divided" => match read_word("", lexer).as_str() {
207
+
"divided" => match read_word(lexer).as_str() {
186
208
"by" => Token::Operator(Divide),
187
209
string => return Err(format!("Invalid string: {}", string)),
188
210
},
···
243
265
"mi" | "mile" | "miles" => Token::Unit(Mile),
244
266
"marathon" | "marathons" => Token::Unit(Marathon),
245
267
"nmi" => Token::Unit(NauticalMile),
246
-
"nautical" => match read_word("", lexer).as_str() {
268
+
"nautical" => match read_word(lexer).as_str() {
247
269
"mile" | "miles" => Token::Unit(NauticalMile),
248
270
string => return Err(format!("Invalid string: {}", string)),
249
271
},
250
272
"ly" | "lightyear" | "lightyears" => Token::Unit(LightYear),
251
273
"lightsec" | "lightsecs" | "lightsecond" | "lightseconds" => Token::Unit(LightSecond),
252
-
"light" => match read_word("", lexer).as_str() {
274
+
"light" => match read_word(lexer).as_str() {
253
275
"yr" | "yrs" | "year" | "years" => Token::Unit(LightYear),
254
276
"sec" | "secs" | "second" | "seconds" => Token::Unit(LightSecond),
255
277
string => return Err(format!("Invalid string: {}", string)),
···
272
294
"sqft" | "ft2" | "foot2" | "feet2" => Token::Unit(SquareFoot),
273
295
"sqyd" | "yd2" | "yard2" | "yards2" => Token::Unit(SquareYard),
274
296
"sqmi" | "mi2" | "mile2" | "miles2" => Token::Unit(SquareMile),
275
-
"sq" | "square" => match read_word("", lexer).as_str() {
297
+
"sq" | "square" => match read_word(lexer).as_str() {
276
298
"mm" | "millimeter" | "millimeters" | "millimetre" | "millimetres" => {
277
299
Token::Unit(SquareMillimeter)
278
300
}
···
314
336
"ft3" | "foot3" | "feet3" => Token::Unit(CubicFoot),
315
337
"yd3" | "yard3" | "yards3" => Token::Unit(CubicYard),
316
338
"mi3" | "mile3" | "miles3" => Token::Unit(CubicMile),
317
-
"cubic" => match read_word("", lexer).as_str() {
339
+
"cubic" => match read_word(lexer).as_str() {
318
340
"mm" | "millimeter" | "millimeters" | "millimetre" | "millimetres" => {
319
341
Token::Unit(CubicMillimeter)
320
342
}
···
345
367
"ts" | "tsp" | "tspn" | "tspns" | "teaspoon" | "teaspoons" => Token::Unit(Teaspoon),
346
368
"tbs" | "tbsp" | "tablespoon" | "tablespoons" => Token::Unit(Tablespoon),
347
369
"floz" => Token::Unit(FluidOunce),
348
-
"fl" | "fluid" => match read_word("", lexer).as_str() {
370
+
"fl" | "fluid" => match read_word(lexer).as_str() {
349
371
"oz" | "ounce" | "ounces" => Token::Unit(FluidOunce),
350
372
string => return Err(format!("Invalid string: {}", string)),
351
373
},
···
354
376
"qt" | "quart" | "quarts" => Token::Unit(Quart),
355
377
"gal" | "gallon" | "gallons" => Token::Unit(Gallon),
356
378
"bbl" => Token::Unit(OilBarrel),
357
-
"oil" => match read_word("", lexer).as_str() {
379
+
"oil" => match read_word(lexer).as_str() {
358
380
"barrel" | "barrels" => Token::Unit(OilBarrel),
359
381
string => return Err(format!("Invalid string: {}", string)),
360
382
},
361
383
362
-
"metric" => match read_word("", lexer).as_str() {
384
+
"metric" => match read_word(lexer).as_str() {
363
385
"ton" | "tons" | "tonne" | "tonnes" => Token::Unit(MetricTon),
364
386
"hp" | "hps" | "horsepower" | "horsepowers" => Token::Unit(MetricHorsepower),
365
387
string => return Err(format!("Invalid string: {}", string)),
···
372
394
"t" | "tonne" | "tonnes" => Token::Unit(MetricTon),
373
395
"oz" | "ounces" => Token::Unit(Ounce),
374
396
"lb" | "lbs" => Token::Unit(Pound),
375
-
"pound" | "pounds" => match lexer.chars.next() {
376
-
Some("-") => match read_word_plain(&mut lexer.chars).as_str() {
397
+
"pound" | "pounds" => match read_immediate_grapheme("-", lexer) {
398
+
true => match lexer.read_immediate_word().as_str() {
377
399
"force" => Token::LexerKeyword(PoundForce),
378
400
other => {
379
401
lexer.tokens.push(Token::Unit(Pound));
380
402
lexer.tokens.push(Token::Operator(Minus));
381
-
parse_word_if_non_empty(other, lexer)?;
403
+
lex_word_if_non_empty(other, lexer)?;
382
404
return Ok(());
383
405
}
384
406
},
385
-
Some(c) => {
386
-
lexer.tokens.push(Token::Unit(Pound));
387
-
parse_token(c, lexer)?;
388
-
return Ok(());
389
-
}
390
-
None => {
391
-
lexer.tokens.push(Token::Unit(Pound));
392
-
return Ok(());
393
-
}
407
+
false => Token::Unit(Pound),
394
408
},
395
409
"stone" | "stones" => Token::Unit(Stone),
396
410
"st" | "ton" | "tons" => Token::Unit(ShortTon),
397
-
"short" => match read_word("", lexer).as_str() {
411
+
"short" => match read_word(lexer).as_str() {
398
412
"ton" | "tons" | "tonne" | "tonnes" => Token::Unit(ShortTon),
399
413
string => return Err(format!("Invalid string: {}", string)),
400
414
},
401
415
"lt" => Token::Unit(LongTon),
402
-
"long" => match read_word("", lexer).as_str() {
416
+
"long" => match read_word(lexer).as_str() {
403
417
"ton" | "tons" | "tonne" | "tonnes" => Token::Unit(LongTon),
404
418
string => return Err(format!("Invalid string: {}", string)),
405
419
},
···
476
490
"millijoule" | "millijoules" => Token::Unit(Millijoule),
477
491
"j" | "joule" | "joules" => Token::Unit(Joule),
478
492
"nm" => Token::Unit(NewtonMeter),
479
-
"newton" => match lexer.chars.next() {
480
-
Some("-") => match read_word_plain(&mut lexer.chars).as_str() {
493
+
"newton" => match read_immediate_grapheme("-", lexer) {
494
+
true => match lexer.read_immediate_word().as_str() {
481
495
"meter" | "meters" | "metre" | "metres" => Token::Unit(NewtonMeter),
482
496
string => return Err(format!("Invalid string: {}", string)),
483
497
},
484
-
Some(c) => match read_word(c, lexer).as_str() {
498
+
false => match lexer.read_word().as_str() {
485
499
"meter" | "meters" | "metre" | "metres" => Token::Unit(NewtonMeter),
486
500
string => return Err(format!("Invalid string: {}", string)),
487
501
},
488
-
None => return Err(format!("Invalid string: {}", word)),
489
502
},
490
503
"kj" | "kilojoule" | "kilojoules" => Token::Unit(Kilojoule),
491
504
"mj" | "megajoule" | "megajoules" => Token::Unit(Megajoule),
···
494
507
"cal" | "calorie" | "calories" => Token::Unit(Calorie),
495
508
"kcal" | "kilocalorie" | "kilocalories" => Token::Unit(KiloCalorie),
496
509
"btu" => Token::Unit(BritishThermalUnit),
497
-
"british" => match read_word("", lexer).as_str() {
498
-
"thermal" => match read_word("", lexer).as_str() {
510
+
"british" => match read_word(lexer).as_str() {
511
+
"thermal" => match read_word(lexer).as_str() {
499
512
"unit" | "units" => Token::Unit(BritishThermalUnit),
500
513
string => return Err(format!("Invalid string: {}", string)),
501
514
},
···
518
531
"hp" | "hps" | "horsepower" | "horsepowers" => Token::Unit(Horsepower),
519
532
"mhp" | "hpm" => Token::Unit(MetricHorsepower),
520
533
521
-
"watt" => match read_word("", lexer).as_str() {
534
+
"watt" => match read_word(lexer).as_str() {
522
535
"hr" | "hrs" | "hour" | "hours" => Token::Unit(WattHour),
523
536
other => {
524
537
lexer.tokens.push(Token::Unit(Watt));
525
-
parse_word_if_non_empty(other, lexer)?;
538
+
lex_word_if_non_empty(other, lexer)?;
526
539
return Ok(());
527
540
}
528
541
},
529
-
"kilowatt" => match read_word("", lexer).as_str() {
542
+
"kilowatt" => match read_word(lexer).as_str() {
530
543
"hr" | "hrs" | "hour" | "hours" => Token::Unit(KilowattHour),
531
544
other => {
532
545
lexer.tokens.push(Token::Unit(Kilowatt));
533
-
parse_word_if_non_empty(other, lexer)?;
546
+
lex_word_if_non_empty(other, lexer)?;
534
547
return Ok(());
535
548
}
536
549
},
537
-
"megawatt" => match read_word("", lexer).as_str() {
550
+
"megawatt" => match read_word(lexer).as_str() {
538
551
"hr" | "hrs" | "hour" | "hours" => Token::Unit(MegawattHour),
539
552
other => {
540
553
lexer.tokens.push(Token::Unit(Megawatt));
541
-
parse_word_if_non_empty(other, lexer)?;
554
+
lex_word_if_non_empty(other, lexer)?;
542
555
return Ok(());
543
556
}
544
557
},
545
-
"gigawatt" => match read_word("", lexer).as_str() {
558
+
"gigawatt" => match read_word(lexer).as_str() {
546
559
"hr" | "hrs" | "hour" | "hours" => Token::Unit(GigawattHour),
547
560
other => {
548
561
lexer.tokens.push(Token::Unit(Gigawatt));
549
-
parse_word_if_non_empty(other, lexer)?;
562
+
lex_word_if_non_empty(other, lexer)?;
550
563
return Ok(());
551
564
}
552
565
},
553
-
"terawatt" => match read_word("", lexer).as_str() {
566
+
"terawatt" => match read_word(lexer).as_str() {
554
567
"hr" | "hrs" | "hour" | "hours" => Token::Unit(TerawattHour),
555
568
other => {
556
569
lexer.tokens.push(Token::Unit(Terawatt));
557
-
parse_word_if_non_empty(other, lexer)?;
570
+
lex_word_if_non_empty(other, lexer)?;
558
571
return Ok(());
559
572
}
560
573
},
561
-
"petawatt" => match read_word("", lexer).as_str() {
574
+
"petawatt" => match &*read_word(lexer) {
562
575
"hr" | "hrs" | "hour" | "hours" => Token::Unit(PetawattHour),
563
576
other => {
564
577
lexer.tokens.push(Token::Unit(Petawatt));
565
-
parse_word_if_non_empty(other, lexer)?;
578
+
lex_word_if_non_empty(other, lexer)?;
566
579
return Ok(());
567
580
}
568
581
},
···
626
639
struct Lexer<'a> {
627
640
left_paren_count: u16,
628
641
right_paren_count: u16,
629
-
chars: Peekable<Graphemes<'a>>,
642
+
input: &'a str,
643
+
graphemes: Peekable<GraphemeIndices<'a>>,
630
644
tokens: Vec<Token>,
631
645
}
646
+
impl<'a> Lexer<'a> {
647
+
fn read_word(&mut self) -> String {
648
+
read_word(self)
649
+
}
650
+
fn read_immediate_word(&mut self) -> String {
651
+
read_immediate_word(self)
652
+
}
653
+
fn lex_token(&mut self) -> Result<(), String> {
654
+
lex_token(self)
655
+
}
656
+
}
632
657
633
658
/// Lex an input string and returns [`Token`]s
634
659
pub fn lex(input: &str, remove_trailing_operator: bool) -> Result<Vec<Token>, String> {
···
646
671
let mut lexer = Lexer {
647
672
left_paren_count: 0,
648
673
right_paren_count: 0,
649
-
chars: UnicodeSegmentation::graphemes(input.as_str(), true).peekable(),
674
+
input: &input,
675
+
graphemes: UnicodeSegmentation::grapheme_indices(input.as_str(), true).peekable(),
650
676
tokens: Vec::new(),
651
677
};
652
678
653
-
while let Some(c) = lexer.chars.next() {
654
-
parse_token(c, &mut lexer)?;
679
+
while let Some(_) = lexer.graphemes.peek() {
680
+
lexer.lex_token()?;
655
681
}
656
682
let tokens = &mut lexer.tokens;
657
683
// auto insert missing parentheses in first and last position
···
1024
1050
}
1025
1051
};
1026
1052
let info_msg = format!(
1027
-
"run_lex input: {}\nexpected: {:?}\nreceived: {:?}",
1053
+
"run_lex assertion failed.\n input: {}\n left: {:?}\n right: {:?}",
1028
1054
input, expected_tokens, tokens
1029
1055
);
1030
1056
assert!(tokens == expected_tokens, "{info_msg}");
···
1057
1083
let input_nonplural_units = nonplural_data_units.replace_all(input, "$1");
1058
1084
let tokens_nonplural_units = lex(&input_nonplural_units, false).unwrap();
1059
1085
let info_msg = format!(
1060
-
"run_datarate_lex input: {}\nexpected: {:?}\nreceived: {:?}",
1086
+
"run_datarate_lex input: {}\n left: {:?}\n right: {:?}",
1061
1087
input, expected_tokens, tokens_nonplural_units
1062
1088
);
1063
1089
assert!(tokens_nonplural_units == expected_tokens, "{info_msg}");