feat: Enhance #use directive for directories and cwd
The `#use` directive now supports referencing entire directories, automatically including all `.myc` files within them. Additionally, environments are now initialized with the current working directory as a default search path, simplifying module resolution.
This commit is contained in:
@@ -20,20 +20,21 @@ We need a way to explicitly define dependencies *within* a Myc script, ensuring
|
|||||||
|
|
||||||
## The Solution: `#use` Preprocessor Directives
|
## The Solution: `#use` Preprocessor Directives
|
||||||
|
|
||||||
We introduce a preprocessor directive `#use` that is evaluated *before* the main parsing phase begins.
|
We introduce a preprocessor directive `#use` that is evaluated *before* the main parsing phase begins. It can target either single files or entire directories.
|
||||||
|
|
||||||
### Syntax
|
### Syntax
|
||||||
|
|
||||||
```myc
|
```myc
|
||||||
#use math->indicators
|
#use math->indicators
|
||||||
#use utils
|
#use utils
|
||||||
|
#use mylib
|
||||||
|
|
||||||
(do
|
(do
|
||||||
;; Actual Myc code begins here
|
;; Actual Myc code begins here
|
||||||
(def my_var (indicators.calculate ...))
|
(def my_var (indicators.calculate ...))
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
*(Here `math->indicators` resolves to `math/indicators.myc` and `utils` resolves to `utils.myc`)*
|
*(Here `math->indicators` resolves to the file `math/indicators.myc`. `utils` resolves to `utils.myc`. If `mylib` is a directory instead of a file, it resolves to all `.myc` files within that directory.)*
|
||||||
|
|
||||||
### Implementation Plan
|
### Implementation Plan
|
||||||
|
|
||||||
@@ -41,13 +42,15 @@ We introduce a preprocessor directive `#use` that is evaluated *before* the main
|
|||||||
The `Environment` will be extended with a `HashSet<PathBuf>` (e.g., `loaded_modules`) to keep track of canonical paths that have already been loaded, ensuring idempotency.
|
The `Environment` will be extended with a `HashSet<PathBuf>` (e.g., `loaded_modules`) to keep track of canonical paths that have already been loaded, ensuring idempotency.
|
||||||
|
|
||||||
2. **Pre-Parse Extraction:**
|
2. **Pre-Parse Extraction:**
|
||||||
Before the AST `Parser` processes a script, a lightweight scraper will read the top lines of the file. It will extract all `#use` paths and stop as soon as it encounters a line that is not empty, not a comment (`;`), and not a `#use` directive.
|
Before the AST `Parser` processes a script, a lightweight scraper will read the top lines of the file. It will extract all `#use` paths and stop as soon as it encounters a line that is not empty, not a comment (`;` or `#`), and not a `#use` directive.
|
||||||
|
|
||||||
3. **Recursive Resolution:**
|
3. **Recursive Resolution:**
|
||||||
- The scraper extracts the module path, replaces `->` with the OS-specific directory separator, and appends `.myc`.
|
- The scraper extracts the module path and replaces `->` with the OS-specific directory separator.
|
||||||
- Paths are resolved relative to the file that declares the dependency.
|
- It first checks if `<path>.myc` exists as a file. If it does, it resolves to that single file.
|
||||||
|
- If not, it checks if `<path>` is a directory. If so, it resolves to all `.myc` files inside that directory (sorted alphabetically).
|
||||||
|
- Paths are resolved relative to the file that declares the dependency, with a fallback to the global `--lib` search paths.
|
||||||
- If a path is not in `loaded_modules`, it is added to the set.
|
- If a path is not in `loaded_modules`, it is added to the set.
|
||||||
- The file is read, and the extraction process runs recursively on its content.
|
- The files are read, and the extraction process runs recursively on their content.
|
||||||
- This creates a flat list of all dependencies in a valid topological order.
|
- This creates a flat list of all dependencies in a valid topological order.
|
||||||
|
|
||||||
4. **Two-Pass Compilation:**
|
4. **Two-Pass Compilation:**
|
||||||
|
|||||||
+2
-3
@@ -1,10 +1,9 @@
|
|||||||
#use lib->sma
|
#use rtl
|
||||||
;; Output: void
|
|
||||||
|
|
||||||
(do
|
(do
|
||||||
(def sma20 (SMA 20))
|
(def sma20 (SMA 20))
|
||||||
|
|
||||||
(def n 1000)
|
(def n 100)
|
||||||
(while (> n 0)
|
(while (> n 0)
|
||||||
(do
|
(do
|
||||||
(print "val=" n " sma20=" (sma20 n))
|
(print "val=" n " sma20=" (sma20 n))
|
||||||
|
|||||||
+73
-35
@@ -173,6 +173,15 @@ impl Environment {
|
|||||||
env
|
env
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Adds the current working directory to the global search paths.
|
||||||
|
/// This is a builder method to easily initialize an environment for file-based usage.
|
||||||
|
pub fn with_cwd(self) -> Self {
|
||||||
|
if let Ok(cwd) = std::env::current_dir() {
|
||||||
|
self.add_search_path(cwd);
|
||||||
|
}
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub fn add_search_path(&self, path: impl AsRef<Path>) {
|
pub fn add_search_path(&self, path: impl AsRef<Path>) {
|
||||||
self.search_paths.borrow_mut().push(path.as_ref().to_path_buf());
|
self.search_paths.borrow_mut().push(path.as_ref().to_path_buf());
|
||||||
}
|
}
|
||||||
@@ -215,27 +224,54 @@ impl Environment {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Resolves a #use module path relative to a base path, then falls back to search paths.
|
/// Resolves a #use module path relative to a base path, then falls back to search paths.
|
||||||
fn resolve_module_path(&self, module_path: &str, base_path: &Path) -> Result<PathBuf, String> {
|
/// Returns a list of all matching .myc files (single file, or all files in a directory).
|
||||||
|
fn resolve_module_paths(&self, module_path: &str, base_path: &Path) -> Result<Vec<PathBuf>, String> {
|
||||||
let relative_path = module_path.replace("->", std::path::MAIN_SEPARATOR_STR);
|
let relative_path = module_path.replace("->", std::path::MAIN_SEPARATOR_STR);
|
||||||
let file_name = format!("{}.myc", relative_path);
|
let file_name = format!("{}.myc", relative_path);
|
||||||
|
|
||||||
|
let check_path = |base: &Path| -> Option<Vec<PathBuf>> {
|
||||||
|
// Check for file first
|
||||||
|
let mut file_p = base.to_path_buf();
|
||||||
|
file_p.push(&file_name);
|
||||||
|
if file_p.is_file() && let Ok(canon) = file_p.canonicalize() {
|
||||||
|
return Some(vec![canon]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for directory
|
||||||
|
let mut dir_p = base.to_path_buf();
|
||||||
|
dir_p.push(&relative_path);
|
||||||
|
if dir_p.is_dir() {
|
||||||
|
let mut files = Vec::new();
|
||||||
|
if let Ok(entries) = std::fs::read_dir(&dir_p) {
|
||||||
|
let mut valid_entries: Vec<_> = entries.filter_map(|e| e.ok()).collect();
|
||||||
|
valid_entries.sort_by_key(|e| e.path());
|
||||||
|
for entry in valid_entries {
|
||||||
|
let p = entry.path();
|
||||||
|
if p.is_file() && p.extension().is_some_and(|ext| ext == "myc") && let Ok(canon) = p.canonicalize() {
|
||||||
|
files.push(canon);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !files.is_empty() {
|
||||||
|
return Some(files);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
// 1. Try relative to base_path
|
// 1. Try relative to base_path
|
||||||
let mut path = base_path.to_path_buf();
|
if let Some(paths) = check_path(base_path) {
|
||||||
path.push(&file_name);
|
return Ok(paths);
|
||||||
if path.is_file() {
|
|
||||||
return path.canonicalize().map_err(|e| format!("IO Error: {}", e));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Try search paths
|
// 2. Try search paths
|
||||||
for sp in self.search_paths.borrow().iter() {
|
for sp in self.search_paths.borrow().iter() {
|
||||||
let mut path = sp.clone();
|
if let Some(paths) = check_path(sp) {
|
||||||
path.push(&file_name);
|
return Ok(paths);
|
||||||
if path.is_file() {
|
|
||||||
return path.canonicalize().map_err(|e| format!("IO Error: {}", e));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(format!("Could not find module '{}' (looked for {})", module_path, file_name))
|
Err(format!("Could not find module or directory '{}'", module_path))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Recursively collects all dependencies starting from a given source string.
|
/// Recursively collects all dependencies starting from a given source string.
|
||||||
@@ -248,33 +284,35 @@ impl Environment {
|
|||||||
let directives = self.extract_use_directives(source);
|
let directives = self.extract_use_directives(source);
|
||||||
|
|
||||||
for module_path in directives {
|
for module_path in directives {
|
||||||
let abs_path = self.resolve_module_path(&module_path, base_path)?;
|
let abs_paths = self.resolve_module_paths(&module_path, base_path)?;
|
||||||
|
|
||||||
if self.loaded_modules.borrow().contains(&abs_path) {
|
for abs_path in abs_paths {
|
||||||
continue;
|
if self.loaded_modules.borrow().contains(&abs_path) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.loaded_modules.borrow_mut().insert(abs_path.clone());
|
||||||
|
|
||||||
|
let lib_source = std::fs::read_to_string(&abs_path)
|
||||||
|
.map_err(|e| format!("Failed to read module {:?}: {}", abs_path, e))?;
|
||||||
|
|
||||||
|
// Pre-process dependencies of the library BEFORE adding it, ensuring topological order
|
||||||
|
let lib_base = abs_path.parent().unwrap_or(Path::new("."));
|
||||||
|
self.collect_dependencies(&lib_source, lib_base, all_files)?;
|
||||||
|
|
||||||
|
let mut parser = Parser::new(&lib_source);
|
||||||
|
let untyped_ast = parser.parse_expression();
|
||||||
|
|
||||||
|
if parser.diagnostics.has_errors() {
|
||||||
|
return Err(format!(
|
||||||
|
"Parser error in module {:?}:\n{}",
|
||||||
|
abs_path,
|
||||||
|
parser.diagnostics.items[0].message
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
all_files.push((abs_path, untyped_ast));
|
||||||
}
|
}
|
||||||
|
|
||||||
self.loaded_modules.borrow_mut().insert(abs_path.clone());
|
|
||||||
|
|
||||||
let lib_source = std::fs::read_to_string(&abs_path)
|
|
||||||
.map_err(|e| format!("Failed to read module {:?}: {}", abs_path, e))?;
|
|
||||||
|
|
||||||
// Pre-process dependencies of the library BEFORE adding it, ensuring topological order
|
|
||||||
let lib_base = abs_path.parent().unwrap_or(Path::new("."));
|
|
||||||
self.collect_dependencies(&lib_source, lib_base, all_files)?;
|
|
||||||
|
|
||||||
let mut parser = Parser::new(&lib_source);
|
|
||||||
let untyped_ast = parser.parse_expression();
|
|
||||||
|
|
||||||
if parser.diagnostics.has_errors() {
|
|
||||||
return Err(format!(
|
|
||||||
"Parser error in module {:?}:\n{}",
|
|
||||||
abs_path,
|
|
||||||
parser.diagnostics.items[0].message
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
all_files.push((abs_path, untyped_ast));
|
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -42,9 +42,9 @@ struct Cli {
|
|||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
let mut env = Environment::new();
|
let mut env = Environment::new().with_cwd();
|
||||||
env.optimization = !cli.no_opt;
|
env.optimization = !cli.no_opt;
|
||||||
|
|
||||||
// Load libraries (Now just search paths)
|
// Load libraries (Now just search paths)
|
||||||
for lib_path in &cli.lib {
|
for lib_path in &cli.lib {
|
||||||
env.add_search_path(lib_path);
|
env.add_search_path(lib_path);
|
||||||
|
|||||||
+5
-3
@@ -61,6 +61,8 @@ impl Default for CompilerApp {
|
|||||||
})
|
})
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let env = Environment::new().with_cwd();
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
source_code: String::from(
|
source_code: String::from(
|
||||||
r#"
|
r#"
|
||||||
@@ -81,7 +83,7 @@ impl Default for CompilerApp {
|
|||||||
trace_logs: Vec::new(),
|
trace_logs: Vec::new(),
|
||||||
active_tab: AppTab::Output,
|
active_tab: AppTab::Output,
|
||||||
debug_enabled: false,
|
debug_enabled: false,
|
||||||
env: Environment::new(),
|
env,
|
||||||
is_first_frame: true,
|
is_first_frame: true,
|
||||||
examples,
|
examples,
|
||||||
}
|
}
|
||||||
@@ -94,7 +96,7 @@ impl CompilerApp {
|
|||||||
self.trace_logs.clear();
|
self.trace_logs.clear();
|
||||||
|
|
||||||
// Reset environment for a fresh run
|
// Reset environment for a fresh run
|
||||||
self.env = Environment::new();
|
self.env = Environment::new().with_cwd();
|
||||||
self.env.optimization = true;
|
self.env.optimization = true;
|
||||||
|
|
||||||
let res = (|| -> Result<(myc::ast::types::Value, std::time::Duration), String> {
|
let res = (|| -> Result<(myc::ast::types::Value, std::time::Duration), String> {
|
||||||
@@ -160,7 +162,7 @@ impl CompilerApp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn dump_ast(&mut self) {
|
fn dump_ast(&mut self) {
|
||||||
self.env = Environment::new();
|
self.env = Environment::new().with_cwd();
|
||||||
self.env.optimization = true; // Enable optimization for AST dump
|
self.env.optimization = true; // Enable optimization for AST dump
|
||||||
|
|
||||||
if let Err(e) = self.env.preload_dependencies(&self.source_code, self.current_example_path.as_deref()) {
|
if let Err(e) = self.env.preload_dependencies(&self.source_code, self.current_example_path.as_deref()) {
|
||||||
|
|||||||
+3
-6
@@ -27,8 +27,7 @@ pub fn run_functional_tests_with_optimization(enabled: bool) -> Vec<TestResult>
|
|||||||
let output_re = Regex::new(r";; Output: (.*)").unwrap();
|
let output_re = Regex::new(r";; Output: (.*)").unwrap();
|
||||||
|
|
||||||
for entry in entries.filter_map(|e| e.ok()) {
|
for entry in entries.filter_map(|e| e.ok()) {
|
||||||
let mut env = Environment::new(); // Fresh environment per test file
|
let mut env = Environment::new().with_cwd(); // Fresh environment per test file
|
||||||
env.add_search_path(".");
|
|
||||||
env.optimization = enabled;
|
env.optimization = enabled;
|
||||||
let path = entry.path();
|
let path = entry.path();
|
||||||
if path.extension().is_some_and(|ext| ext == "myc") {
|
if path.extension().is_some_and(|ext| ext == "myc") {
|
||||||
@@ -121,8 +120,7 @@ pub fn run_benchmarks(update: bool, filter: Option<&str>) -> Vec<BenchmarkResult
|
|||||||
.unwrap_or(1);
|
.unwrap_or(1);
|
||||||
|
|
||||||
// Compile once for this file (symbols/types are compatible with all fresh environments)
|
// Compile once for this file (symbols/types are compatible with all fresh environments)
|
||||||
let initial_env = Environment::new();
|
let initial_env = Environment::new().with_cwd();
|
||||||
initial_env.add_search_path(".");
|
|
||||||
|
|
||||||
if let Err(e) = initial_env.preload_dependencies(&content, Some(&path)) {
|
if let Err(e) = initial_env.preload_dependencies(&content, Some(&path)) {
|
||||||
results.push(BenchmarkResult {
|
results.push(BenchmarkResult {
|
||||||
@@ -152,8 +150,7 @@ pub fn run_benchmarks(update: bool, filter: Option<&str>) -> Vec<BenchmarkResult
|
|||||||
// Helper to measure sum of VM execution times over N executions in one environment
|
// Helper to measure sum of VM execution times over N executions in one environment
|
||||||
let measure_sum =
|
let measure_sum =
|
||||||
|n: u32, node: &crate::ast::compiler::TypedNode| -> Result<Duration, String> {
|
|n: u32, node: &crate::ast::compiler::TypedNode| -> Result<Duration, String> {
|
||||||
let env = Environment::new();
|
let env = Environment::new().with_cwd();
|
||||||
env.add_search_path(".");
|
|
||||||
if let Err(e) = env.preload_dependencies(&content, Some(&path)) {
|
if let Err(e) = env.preload_dependencies(&content, Some(&path)) {
|
||||||
return Err(format!("Dependency Error in measurement: {}", e));
|
return Err(format!("Dependency Error in measurement: {}", e));
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user