feat: init line tokens

scie-grammar/src/grammar/line_tokens.rs

+#[derive(Debug, Clone)]
+pub struct TokenTypeMatcher {}
+
+#[derive(Debug, Clone)]
+pub struct IToken {
+    pub start_index: i32,
+    pub end_index: i32,
+    pub scopes: Vec<String>
+}
+
+#[derive(Debug, Clone)]
+pub struct LineTokens {
+    pub emit_binary_tokens: bool,
+    pub _line_text: String,
+    pub _tokens: Vec<IToken>,
+    pub _binary_tokens: Vec<IToken>,
+    pub _last_token_end_index: i32,
+    pub _token_type_overrides: Vec<TokenTypeMatcher>
+}
+
+impl LineTokens {
+    pub fn new(emit_binary_tokens: bool, _line_text: String, _token_type_overrides: Vec<TokenTypeMatcher>) -> Self {
+        LineTokens {
+            emit_binary_tokens,
+            _line_text,
+            _tokens: vec![],
+            _binary_tokens: vec![],
+            _last_token_end_index: 0,
+            _token_type_overrides
+        }
+    }
+}
+

scie-grammar/src/grammar/mod.rs

 pub mod grammar;
 pub mod grammar_reader;
+pub mod line_tokens;

scie-grammar/src/lib.rs

@@ -9,5 +9,5 @@ extern crate erased_serde;
 pub mod grammar;
 pub mod grammar_registry;
 pub mod inter;
-pub mod reg_exp_source;
 pub mod rule;
+pub mod matcher;

scie-grammar/src/matcher.rs

+pub enum MatchPriority {
+    High,
+    Medium,
+    Low,
+}
+
+pub struct MatcherWithPriority {
+    matcher: Matcher,
+    priority: MatchPriority,
+}
+
+pub struct Matcher {
+
+}
\ No newline at end of file

scie-grammar/src/rule/mod.rs

 pub mod rule_factory;
 pub mod rule_convert;
+pub mod reg_exp_source;
 
 use crate::inter::{ILocation, IRawGrammar, IRawRepository};
-use crate::reg_exp_source::{RegExpSource, RegExpSourceList};
+use reg_exp_source::{RegExpSource, RegExpSourceList};
 use crate::rule::rule_factory::ICompilePatternsResult;
 use core::fmt;
 use dyn_clone::{clone_trait_object, DynClone};

vscode-tests/vendor/main.js

@@ -1509,7 +1509,7 @@ var LineTokens = /** @class */ (function () {
         this._tokens.push({
             startIndex: this._lastTokenEndIndex,
             endIndex: endIndex,
-            // value: lineText.substring(lastTokenEndIndex, endIndex),
+            // value: lineText.substring(lastTokenEndIndex, end_index),
             scopes: scopes
         });
         this._lastTokenEndIndex = endIndex;