Skip to main content

max / goingson

23.0 KB · 696 lines History Blame Raw
1 //! Native CSV/TSV import commands.
2 //!
3 //! GoingsOn imports CSV as its interchange format: platform-specific exports
4 //! (Todoist, Things, etc.) are converted to CSV by separate tools, then brought
5 //! in here. This replaces the former Rhai plugin runtime, the parsing and field
6 //! mapping that lived in a sandboxed `.rhai` script are now native Rust, so there
7 //! is no plugin sandbox, capability model, or provenance surface to secure.
8 //!
9 //! Flow: `preview_import` parses the file into typed items (dry run);
10 //! `execute_import` creates the entities. Entity type (task/project/event) is
11 //! auto-detected from the header columns.
12
13 use std::collections::{HashMap, HashSet};
14 use std::sync::Arc;
15
16 use serde::Deserialize;
17 use tauri::State;
18 use tracing::instrument;
19
20 use goingson_core::{
21 ImportEntityType, ImportEventData, ImportExecuteResult, ImportFailure, ImportItem,
22 ImportItemData, ImportOptions, ImportParseResult, ImportProjectData, ImportTaskData,
23 NewEventBuilder, NewProject, NewTaskBuilder, ParseableEnum, Priority, ProjectId,
24 };
25
26 use super::error::ApiError;
27 use super::import_external::read_import_file;
28 use crate::state::{AppState, DESKTOP_USER_ID};
29
30 /// Input for previewing a CSV import (dry run, no DB writes).
31 #[derive(Debug, Deserialize)]
32 #[serde(rename_all = "camelCase")]
33 pub struct PreviewImportInput {
34 pub file_path: String,
35 #[serde(default)]
36 pub options: ImportOptions,
37 }
38
39 /// Input for executing a CSV import.
40 #[derive(Debug, Deserialize)]
41 #[serde(rename_all = "camelCase")]
42 pub struct ExecuteImportInput {
43 pub file_path: String,
44 #[serde(default)]
45 pub options: ImportOptions,
46 /// Indices of items to import (all if empty).
47 #[serde(default)]
48 pub selected_indices: Vec<usize>,
49 }
50
51 /// Previews a CSV/TSV import by parsing the file without creating entities.
52 #[tauri::command]
53 #[instrument(skip_all)]
54 pub async fn preview_import(input: PreviewImportInput) -> Result<ImportParseResult, ApiError> {
55 let content = read_import_file(&input.file_path)?;
56 parse_csv_import(&content, &input.options)
57 }
58
59 /// Executes a CSV/TSV import, creating entities in the database.
60 #[tauri::command]
61 #[instrument(skip_all)]
62 pub async fn execute_import(
63 state: State<'_, Arc<AppState>>,
64 input: ExecuteImportInput,
65 ) -> Result<ImportExecuteResult, ApiError> {
66 let content = read_import_file(&input.file_path)?;
67 let parsed = parse_csv_import(&content, &input.options)?;
68
69 let projects = state
70 .projects
71 .list_all(DESKTOP_USER_ID)
72 .await
73 .map_err(ApiError::from)?;
74 let project_list: Vec<(String, String)> = projects
75 .iter()
76 .map(|p| (p.id.to_string(), p.name.clone()))
77 .collect();
78
79 let items: Vec<&ImportItem> = if input.selected_indices.is_empty() {
80 parsed.items.iter().collect()
81 } else {
82 parsed
83 .items
84 .iter()
85 .enumerate()
86 .filter(|(idx, _)| input.selected_indices.contains(idx))
87 .map(|(_, item)| item)
88 .collect()
89 };
90
91 match parsed.entity_type {
92 ImportEntityType::Task => import_tasks(&state, &items, &project_list).await,
93 ImportEntityType::Project => import_projects(&state, &items).await,
94 ImportEntityType::Event => import_events(&state, &items, &project_list).await,
95 }
96 }
97
98 // CSV Parsing
99
100 /// Parses CSV/TSV content into typed import items, auto-detecting the entity
101 /// type from the header row.
102 fn parse_csv_import(content: &str, options: &ImportOptions) -> Result<ImportParseResult, ApiError> {
103 let rows = parse_csv_rows(content, options)?;
104
105 if rows.is_empty() {
106 return Ok(ImportParseResult {
107 entity_type: ImportEntityType::Task,
108 items: Vec::new(),
109 warnings: vec!["The file contained no data rows.".to_string()],
110 });
111 }
112
113 let columns: HashSet<&str> = rows[0].keys().map(std::string::String::as_str).collect();
114 let entity_type = detect_entity_type(&columns);
115
116 let (items, warnings) = match entity_type {
117 ImportEntityType::Task => parse_tasks(&rows),
118 ImportEntityType::Project => parse_projects(&rows),
119 ImportEntityType::Event => parse_events(&rows),
120 };
121
122 Ok(ImportParseResult {
123 entity_type,
124 items,
125 warnings,
126 })
127 }
128
129 /// Reads CSV/TSV rows into maps keyed by lowercased header (case-insensitive
130 /// field matching). Strips a leading BOM (Excel exports). With no header row,
131 /// columns are keyed `col_0`, `col_1`, ...
132 fn parse_csv_rows(
133 content: &str,
134 options: &ImportOptions,
135 ) -> Result<Vec<HashMap<String, String>>, ApiError> {
136 let content = content.strip_prefix('\u{FEFF}').unwrap_or(content);
137 let delimiter = options.delimiter.unwrap_or(',') as u8;
138
139 let mut reader = csv::ReaderBuilder::new()
140 .has_headers(options.has_header)
141 .delimiter(delimiter)
142 .flexible(true)
143 .from_reader(content.as_bytes());
144
145 let headers: Vec<String> = if options.has_header {
146 reader
147 .headers()
148 .map_err(|e| ApiError::validation_msg(format!("Failed to read CSV headers: {e}")))?
149 .iter()
150 .map(|s| s.trim().to_lowercase())
151 .collect()
152 } else {
153 Vec::new()
154 };
155
156 let mut rows = Vec::new();
157 for result in reader.records() {
158 let record =
159 result.map_err(|e| ApiError::validation_msg(format!("CSV parse error: {e}")))?;
160 let mut row = HashMap::new();
161 for (i, field) in record.iter().enumerate() {
162 let key = headers
163 .get(i)
164 .cloned()
165 .unwrap_or_else(|| format!("col_{i}"));
166 row.insert(key, field.to_string());
167 }
168 rows.push(row);
169 }
170 Ok(rows)
171 }
172
173 /// Detects the entity type from the (lowercased) header columns.
174 fn detect_entity_type(columns: &HashSet<&str>) -> ImportEntityType {
175 if columns.contains("start") || columns.contains("start_time") || columns.contains("start_date")
176 {
177 return ImportEntityType::Event;
178 }
179 if columns.contains("project_type")
180 || (columns.contains("name") && !columns.contains("description"))
181 {
182 return ImportEntityType::Project;
183 }
184 ImportEntityType::Task
185 }
186
187 /// First non-empty value among the given (lowercased) candidate column names.
188 fn get_field(row: &HashMap<String, String>, names: &[&str]) -> Option<String> {
189 for name in names {
190 if let Some(value) = row.get(*name) {
191 let trimmed = value.trim();
192 if !trimmed.is_empty() {
193 return Some(trimmed.to_string());
194 }
195 }
196 }
197 None
198 }
199
200 fn parse_tasks(rows: &[HashMap<String, String>]) -> (Vec<ImportItem>, Vec<String>) {
201 let mut items = Vec::new();
202 let mut warnings = Vec::new();
203
204 for (idx, row) in rows.iter().enumerate() {
205 let Some(description) =
206 get_field(row, &["description", "task", "title", "name", "subject"])
207 else {
208 warnings.push(format!("Row {}: missing description, skipped.", idx + 1));
209 continue;
210 };
211
212 let data = ImportTaskData {
213 description,
214 due: get_field(row, &["due", "due_date", "deadline", "date"])
215 .and_then(|d| normalize_date(&d)),
216 priority: normalize_priority(get_field(row, &["priority", "pri", "importance"])),
217 status: normalize_task_status(get_field(row, &["status", "state"])),
218 project_name: get_field(row, &["project", "project_name", "category"]),
219 tags: Some(parse_tags(get_field(
220 row,
221 &["tags", "labels", "categories"],
222 ))),
223 notes: get_field(row, &["notes", "note", "comments", "body"]),
224 };
225
226 items.push(ImportItem {
227 source_index: idx + 1,
228 data: ImportItemData::Task(data),
229 has_errors: false,
230 errors: Vec::new(),
231 });
232 }
233
234 (items, warnings)
235 }
236
237 fn parse_projects(rows: &[HashMap<String, String>]) -> (Vec<ImportItem>, Vec<String>) {
238 let mut items = Vec::new();
239 let mut warnings = Vec::new();
240
241 for (idx, row) in rows.iter().enumerate() {
242 let Some(name) = get_field(row, &["name", "project", "title"]) else {
243 warnings.push(format!("Row {}: missing name, skipped.", idx + 1));
244 continue;
245 };
246
247 let data = ImportProjectData {
248 name,
249 description: get_field(row, &["description", "desc", "notes"]),
250 project_type: normalize_project_type(get_field(
251 row,
252 &["type", "project_type", "category"],
253 )),
254 status: normalize_project_status(get_field(row, &["status", "state"])),
255 };
256
257 items.push(ImportItem {
258 source_index: idx + 1,
259 data: ImportItemData::Project(data),
260 has_errors: false,
261 errors: Vec::new(),
262 });
263 }
264
265 (items, warnings)
266 }
267
268 fn parse_events(rows: &[HashMap<String, String>]) -> (Vec<ImportItem>, Vec<String>) {
269 let mut items = Vec::new();
270 let mut warnings = Vec::new();
271
272 for (idx, row) in rows.iter().enumerate() {
273 let Some(title) = get_field(row, &["title", "name", "event", "subject", "summary"]) else {
274 warnings.push(format!("Row {}: missing title, skipped.", idx + 1));
275 continue;
276 };
277
278 let Some(start_raw) =
279 get_field(row, &["start", "start_time", "start_date", "date", "when"])
280 else {
281 warnings.push(format!("Row {}: missing start time, skipped.", idx + 1));
282 continue;
283 };
284 // Normalize if recognizable, otherwise keep the raw value so the
285 // executor's parse_datetime gets a chance and reports a precise failure.
286 let start = normalize_date(&start_raw).unwrap_or(start_raw);
287
288 let data = ImportEventData {
289 title,
290 start,
291 end: get_field(row, &["end", "end_time", "end_date"]).and_then(|e| normalize_date(&e)),
292 location: get_field(row, &["location", "place", "venue", "where"]),
293 description: get_field(row, &["description", "notes", "body", "details"]),
294 project_name: get_field(row, &["project", "project_name", "category"]),
295 };
296
297 items.push(ImportItem {
298 source_index: idx + 1,
299 data: ImportItemData::Event(data),
300 has_errors: false,
301 errors: Vec::new(),
302 });
303 }
304
305 (items, warnings)
306 }
307
308 /// Maps loose priority spellings to High/Medium/Low. `None` input → `None`.
309 fn normalize_priority(value: Option<String>) -> Option<String> {
310 let value = value?;
311 Some(
312 match value.to_lowercase().as_str() {
313 "high" | "1" | "h" | "urgent" | "critical" => "High",
314 "low" | "3" | "l" | "minor" => "Low",
315 _ => "Medium",
316 }
317 .to_string(),
318 )
319 }
320
321 fn normalize_task_status(value: Option<String>) -> Option<String> {
322 let value = value?;
323 Some(
324 match value.to_lowercase().as_str() {
325 "done" | "complete" | "completed" | "finished" | "closed" => "Done",
326 "in progress" | "inprogress" | "started" | "working" | "active" => "InProgress",
327 _ => "Pending",
328 }
329 .to_string(),
330 )
331 }
332
333 fn normalize_project_type(value: Option<String>) -> Option<String> {
334 let value = value?;
335 Some(
336 match value.to_lowercase().as_str() {
337 "job" | "work" | "employment" => "Job",
338 "side project" | "sideproject" | "personal" | "hobby" => "SideProject",
339 "company" | "business" | "startup" => "Company",
340 "essay" | "writing" => "Essay",
341 "article" | "blog" | "post" => "Article",
342 "painting" | "art" | "visual" => "Painting",
343 _ => "Other",
344 }
345 .to_string(),
346 )
347 }
348
349 fn normalize_project_status(value: Option<String>) -> Option<String> {
350 let value = value?;
351 Some(
352 match value.to_lowercase().as_str() {
353 "on hold" | "onhold" | "paused" | "waiting" => "OnHold",
354 "completed" | "done" | "finished" => "Completed",
355 "archived" | "inactive" | "closed" => "Archived",
356 _ => "Active",
357 }
358 .to_string(),
359 )
360 }
361
362 /// Splits a delimited tag string (`,`, `;`, or `|`) into trimmed, non-empty tags.
363 fn parse_tags(value: Option<String>) -> Vec<String> {
364 match value {
365 None => Vec::new(),
366 Some(v) => v
367 .split([',', ';', '|'])
368 .map(str::trim)
369 .filter(|t| !t.is_empty())
370 .map(std::string::ToString::to_string)
371 .collect(),
372 }
373 }
374
375 /// Normalizes a date/datetime string to ISO form, trying common layouts.
376 /// Returns `None` if no layout matches (caller decides how to handle).
377 fn normalize_date(input: &str) -> Option<String> {
378 let input = input.trim();
379 if input.is_empty() {
380 return None;
381 }
382
383 const FORMATS: [&str; 10] = [
384 "%Y-%m-%d",
385 "%Y-%m-%dT%H:%M:%S",
386 "%Y-%m-%dT%H:%M:%SZ",
387 "%Y-%m-%dT%H:%M:%S%.fZ",
388 "%Y/%m/%d",
389 "%m/%d/%Y",
390 "%d/%m/%Y",
391 "%d-%m-%Y",
392 "%B %d, %Y",
393 "%b %d, %Y",
394 ];
395
396 for format in &FORMATS {
397 if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(input, format) {
398 return Some(dt.format("%Y-%m-%dT%H:%M:%S").to_string());
399 }
400 if let Ok(d) = chrono::NaiveDate::parse_from_str(input, format) {
401 return Some(d.format("%Y-%m-%d").to_string());
402 }
403 }
404 if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(input) {
405 return Some(dt.format("%Y-%m-%dT%H:%M:%S").to_string());
406 }
407 None
408 }
409
410 // Import Executors
411
412 /// Imports parsed task items. Resolves project by name (case-insensitive),
413 /// parses priority and due date; per-item errors are accumulated.
414 async fn import_tasks(
415 state: &AppState,
416 items: &[&ImportItem],
417 projects: &[(String, String)],
418 ) -> Result<ImportExecuteResult, ApiError> {
419 let mut imported = 0;
420 let mut failed = 0;
421 let mut failures = Vec::new();
422
423 for item in items {
424 if let ImportItemData::Task(data) = &item.data {
425 let project_id = resolve_project(data.project_name.as_deref(), projects);
426
427 let priority = data.priority.as_ref().map_or(Priority::Medium, |p| {
428 match p.to_lowercase().as_str() {
429 "high" | "1" => Priority::High,
430 "low" | "3" => Priority::Low,
431 _ => Priority::Medium,
432 }
433 });
434
435 let due = data.due.as_ref().and_then(|d| parse_datetime(d));
436
437 let mut builder = NewTaskBuilder::new(&data.description)
438 .priority(priority)
439 .tags(data.tags.clone().unwrap_or_default());
440 if let Some(d) = due {
441 builder = builder.due(d);
442 }
443 if let Some(pid) = project_id {
444 builder = builder.project_id(pid);
445 }
446
447 match state.tasks.create(DESKTOP_USER_ID, builder.build()).await {
448 Ok(_) => imported += 1,
449 Err(e) => {
450 failed += 1;
451 failures.push(ImportFailure {
452 source_index: item.source_index,
453 message: e.to_string(),
454 });
455 }
456 }
457 }
458 }
459
460 Ok(ImportExecuteResult {
461 imported_count: imported,
462 failed_count: failed,
463 skipped_count: items.len() - imported - failed,
464 failures,
465 })
466 }
467
468 /// Imports parsed project items. Type/status fall back to defaults on
469 /// unrecognized values rather than failing the import.
470 async fn import_projects(
471 state: &AppState,
472 items: &[&ImportItem],
473 ) -> Result<ImportExecuteResult, ApiError> {
474 let mut imported = 0;
475 let mut failed = 0;
476 let mut failures = Vec::new();
477
478 for item in items {
479 if let ImportItemData::Project(data) = &item.data {
480 let new_project = NewProject {
481 name: data.name.clone(),
482 description: data.description.clone().unwrap_or_default(),
483 project_type: data
484 .project_type
485 .as_ref()
486 .map(|t| goingson_core::ProjectType::from_str_or_default(t))
487 .unwrap_or_default(),
488 status: data
489 .status
490 .as_ref()
491 .map(|s| goingson_core::ProjectStatus::from_str_or_default(s))
492 .unwrap_or_default(),
493 };
494
495 match state.projects.create(DESKTOP_USER_ID, new_project).await {
496 Ok(_) => imported += 1,
497 Err(e) => {
498 failed += 1;
499 failures.push(ImportFailure {
500 source_index: item.source_index,
501 message: e.to_string(),
502 });
503 }
504 }
505 }
506 }
507
508 Ok(ImportExecuteResult {
509 imported_count: imported,
510 failed_count: failed,
511 skipped_count: items.len() - imported - failed,
512 failures,
513 })
514 }
515
516 /// Imports parsed event items. `start` is required, events without a parseable
517 /// start time are counted as failures.
518 async fn import_events(
519 state: &AppState,
520 items: &[&ImportItem],
521 projects: &[(String, String)],
522 ) -> Result<ImportExecuteResult, ApiError> {
523 let mut imported = 0;
524 let mut failed = 0;
525 let mut failures = Vec::new();
526
527 for item in items {
528 if let ImportItemData::Event(data) = &item.data {
529 let project_id = resolve_project(data.project_name.as_deref(), projects);
530
531 let Some(start) = parse_datetime(&data.start) else {
532 failed += 1;
533 failures.push(ImportFailure {
534 source_index: item.source_index,
535 message: format!("Invalid start time: {}", data.start),
536 });
537 continue;
538 };
539 let end = data.end.as_ref().and_then(|e| parse_datetime(e));
540
541 let mut builder = NewEventBuilder::new(&data.title, start);
542 if let Some(e) = end {
543 builder = builder.end_time(e);
544 }
545 if let Some(ref loc) = data.location {
546 builder = builder.location(loc);
547 }
548 if let Some(ref desc) = data.description {
549 builder = builder.description(desc);
550 }
551 if let Some(pid) = project_id {
552 builder = builder.project_id(pid);
553 }
554
555 match state.events.create(DESKTOP_USER_ID, builder.build()).await {
556 Ok(_) => imported += 1,
557 Err(e) => {
558 failed += 1;
559 failures.push(ImportFailure {
560 source_index: item.source_index,
561 message: e.to_string(),
562 });
563 }
564 }
565 }
566 }
567
568 Ok(ImportExecuteResult {
569 imported_count: imported,
570 failed_count: failed,
571 skipped_count: items.len() - imported - failed,
572 failures,
573 })
574 }
575
576 /// Resolves a project name (case-insensitive) to its id.
577 fn resolve_project(name: Option<&str>, projects: &[(String, String)]) -> Option<ProjectId> {
578 let name = name?;
579 let name_lower = name.to_lowercase();
580 projects
581 .iter()
582 .find(|(_, n)| n.to_lowercase() == name_lower)
583 .and_then(|(id, _)| uuid::Uuid::parse_str(id).ok().map(ProjectId::from))
584 }
585
586 /// Parses a datetime with a three-format fallback (RFC 3339, ISO without tz
587 /// assumed UTC, date-only at midnight UTC). Returns `None` if none match.
588 fn parse_datetime(s: &str) -> Option<chrono::DateTime<chrono::Utc>> {
589 chrono::DateTime::parse_from_rfc3339(s)
590 .ok()
591 .map(|dt| dt.with_timezone(&chrono::Utc))
592 .or_else(|| {
593 chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S")
594 .ok()
595 .map(|dt| dt.and_utc())
596 })
597 .or_else(|| {
598 chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d")
599 .ok()
600 .map(|d| d.and_hms_opt(0, 0, 0).expect("midnight is valid").and_utc())
601 })
602 }
603
604 #[cfg(test)]
605 mod tests {
606 use super::*;
607
608 fn opts() -> ImportOptions {
609 ImportOptions {
610 has_header: true,
611 delimiter: None,
612 date_format: None,
613 extra: std::collections::HashMap::default(),
614 }
615 }
616
617 #[test]
618 fn detects_event_from_start_column() {
619 let csv = "title,start\nStandup,2024-01-15\n";
620 let r = parse_csv_import(csv, &opts()).unwrap();
621 assert_eq!(r.entity_type, ImportEntityType::Event);
622 assert_eq!(r.items.len(), 1);
623 }
624
625 #[test]
626 fn detects_project_when_name_without_description() {
627 let csv = "name,status\nWebsite,active\n";
628 let r = parse_csv_import(csv, &opts()).unwrap();
629 assert_eq!(r.entity_type, ImportEntityType::Project);
630 }
631
632 #[test]
633 fn defaults_to_task_and_maps_fields() {
634 let csv = "description,priority,tags,project\nBuy milk,high,\"a, b\",Home\n";
635 let r = parse_csv_import(csv, &opts()).unwrap();
636 assert_eq!(r.entity_type, ImportEntityType::Task);
637 let ImportItemData::Task(t) = &r.items[0].data else {
638 panic!("expected task");
639 };
640 assert_eq!(t.description, "Buy milk");
641 assert_eq!(t.priority.as_deref(), Some("High"));
642 assert_eq!(t.project_name.as_deref(), Some("Home"));
643 assert_eq!(
644 t.tags.as_ref().unwrap(),
645 &vec!["a".to_string(), "b".to_string()]
646 );
647 }
648
649 #[test]
650 fn case_insensitive_headers_and_bom_stripped() {
651 let csv = "\u{FEFF}Description,Due\nShip it,2024-03-01\n";
652 let r = parse_csv_import(csv, &opts()).unwrap();
653 let ImportItemData::Task(t) = &r.items[0].data else {
654 panic!("expected task");
655 };
656 assert_eq!(t.description, "Ship it");
657 assert_eq!(t.due.as_deref(), Some("2024-03-01"));
658 }
659
660 #[test]
661 fn rows_missing_required_field_become_warnings() {
662 // Second row has a priority but an empty description: a non-blank
663 // record missing its required field, which is warned and skipped.
664 // (Truly blank lines are dropped by the CSV reader, not warned.)
665 let csv = "description,priority\nReal task,high\n,low\n";
666 let r = parse_csv_import(csv, &opts()).unwrap();
667 assert_eq!(r.items.len(), 1);
668 assert_eq!(r.warnings.len(), 1);
669 }
670
671 #[test]
672 fn empty_file_yields_no_items() {
673 let r = parse_csv_import("", &opts()).unwrap();
674 assert!(r.items.is_empty());
675 }
676
677 #[test]
678 fn tsv_via_delimiter_option() {
679 let mut o = opts();
680 o.delimiter = Some('\t');
681 let csv = "description\tpriority\nTabbed\tlow\n";
682 let r = parse_csv_import(csv, &o).unwrap();
683 let ImportItemData::Task(t) = &r.items[0].data else {
684 panic!("expected task");
685 };
686 assert_eq!(t.description, "Tabbed");
687 assert_eq!(t.priority.as_deref(), Some("Low"));
688 }
689
690 #[test]
691 fn normalize_date_handles_us_format() {
692 assert_eq!(normalize_date("01/15/2024").as_deref(), Some("2024-01-15"));
693 assert_eq!(normalize_date("not a date"), None);
694 }
695 }
696