Skip to main content

max / goingson

23.1 KB · 692 lines History Blame Raw
1 //! Native CSV/TSV import commands.
2 //!
3 //! GoingsOn imports CSV as its interchange format: platform-specific exports
4 //! (Todoist, Things, etc.) are converted to CSV by separate tools, then brought
5 //! in here. This replaces the former Rhai plugin runtime — the parsing and field
6 //! mapping that lived in a sandboxed `.rhai` script are now native Rust, so there
7 //! is no plugin sandbox, capability model, or provenance surface to secure.
8 //!
9 //! Flow: `preview_import` parses the file into typed items (dry run);
10 //! `execute_import` creates the entities. Entity type (task/project/event) is
11 //! auto-detected from the header columns.
12
13 use std::collections::{HashMap, HashSet};
14 use std::sync::Arc;
15
16 use serde::Deserialize;
17 use tauri::State;
18 use tracing::instrument;
19
20 use goingson_core::{
21 ImportEntityType, ImportEventData, ImportExecuteResult, ImportFailure, ImportItem,
22 ImportItemData, ImportOptions, ImportParseResult, ImportProjectData, ImportTaskData,
23 NewEventBuilder, NewProject, NewTaskBuilder, ParseableEnum, Priority, ProjectId,
24 };
25
26 use super::error::ApiError;
27 use super::import_external::read_import_file;
28 use crate::state::{AppState, DESKTOP_USER_ID};
29
30 /// Input for previewing a CSV import (dry run, no DB writes).
31 #[derive(Debug, Deserialize)]
32 #[serde(rename_all = "camelCase")]
33 pub struct PreviewImportInput {
34 pub file_path: String,
35 #[serde(default)]
36 pub options: ImportOptions,
37 }
38
39 /// Input for executing a CSV import.
40 #[derive(Debug, Deserialize)]
41 #[serde(rename_all = "camelCase")]
42 pub struct ExecuteImportInput {
43 pub file_path: String,
44 #[serde(default)]
45 pub options: ImportOptions,
46 /// Indices of items to import (all if empty).
47 #[serde(default)]
48 pub selected_indices: Vec<usize>,
49 }
50
51 /// Previews a CSV/TSV import by parsing the file without creating entities.
52 #[tauri::command]
53 #[instrument(skip_all)]
54 pub async fn preview_import(input: PreviewImportInput) -> Result<ImportParseResult, ApiError> {
55 let content = read_import_file(&input.file_path)?;
56 parse_csv_import(&content, &input.options)
57 }
58
59 /// Executes a CSV/TSV import, creating entities in the database.
60 #[tauri::command]
61 #[instrument(skip_all)]
62 pub async fn execute_import(
63 state: State<'_, Arc<AppState>>,
64 input: ExecuteImportInput,
65 ) -> Result<ImportExecuteResult, ApiError> {
66 let content = read_import_file(&input.file_path)?;
67 let parsed = parse_csv_import(&content, &input.options)?;
68
69 let projects = state
70 .projects
71 .list_all(DESKTOP_USER_ID)
72 .await
73 .map_err(ApiError::from)?;
74 let project_list: Vec<(String, String)> = projects
75 .iter()
76 .map(|p| (p.id.to_string(), p.name.clone()))
77 .collect();
78
79 let items: Vec<&ImportItem> = if input.selected_indices.is_empty() {
80 parsed.items.iter().collect()
81 } else {
82 parsed
83 .items
84 .iter()
85 .enumerate()
86 .filter(|(idx, _)| input.selected_indices.contains(idx))
87 .map(|(_, item)| item)
88 .collect()
89 };
90
91 match parsed.entity_type {
92 ImportEntityType::Task => import_tasks(&state, &items, &project_list).await,
93 ImportEntityType::Project => import_projects(&state, &items).await,
94 ImportEntityType::Event => import_events(&state, &items, &project_list).await,
95 }
96 }
97
98 // ============ CSV Parsing ============
99
100 /// Parses CSV/TSV content into typed import items, auto-detecting the entity
101 /// type from the header row.
102 fn parse_csv_import(content: &str, options: &ImportOptions) -> Result<ImportParseResult, ApiError> {
103 let rows = parse_csv_rows(content, options)?;
104
105 if rows.is_empty() {
106 return Ok(ImportParseResult {
107 entity_type: ImportEntityType::Task,
108 items: Vec::new(),
109 warnings: vec!["The file contained no data rows.".to_string()],
110 });
111 }
112
113 let columns: HashSet<&str> = rows[0].keys().map(|k| k.as_str()).collect();
114 let entity_type = detect_entity_type(&columns);
115
116 let (items, warnings) = match entity_type {
117 ImportEntityType::Task => parse_tasks(&rows),
118 ImportEntityType::Project => parse_projects(&rows),
119 ImportEntityType::Event => parse_events(&rows),
120 };
121
122 Ok(ImportParseResult {
123 entity_type,
124 items,
125 warnings,
126 })
127 }
128
129 /// Reads CSV/TSV rows into maps keyed by lowercased header (case-insensitive
130 /// field matching). Strips a leading BOM (Excel exports). With no header row,
131 /// columns are keyed `col_0`, `col_1`, ...
132 fn parse_csv_rows(
133 content: &str,
134 options: &ImportOptions,
135 ) -> Result<Vec<HashMap<String, String>>, ApiError> {
136 let content = content.strip_prefix('\u{FEFF}').unwrap_or(content);
137 let delimiter = options.delimiter.unwrap_or(',') as u8;
138
139 let mut reader = csv::ReaderBuilder::new()
140 .has_headers(options.has_header)
141 .delimiter(delimiter)
142 .flexible(true)
143 .from_reader(content.as_bytes());
144
145 let headers: Vec<String> = if options.has_header {
146 reader
147 .headers()
148 .map_err(|e| ApiError::validation_msg(format!("Failed to read CSV headers: {}", e)))?
149 .iter()
150 .map(|s| s.trim().to_lowercase())
151 .collect()
152 } else {
153 Vec::new()
154 };
155
156 let mut rows = Vec::new();
157 for result in reader.records() {
158 let record =
159 result.map_err(|e| ApiError::validation_msg(format!("CSV parse error: {}", e)))?;
160 let mut row = HashMap::new();
161 for (i, field) in record.iter().enumerate() {
162 let key = headers
163 .get(i)
164 .cloned()
165 .unwrap_or_else(|| format!("col_{}", i));
166 row.insert(key, field.to_string());
167 }
168 rows.push(row);
169 }
170 Ok(rows)
171 }
172
173 /// Detects the entity type from the (lowercased) header columns.
174 fn detect_entity_type(columns: &HashSet<&str>) -> ImportEntityType {
175 if columns.contains("start") || columns.contains("start_time") || columns.contains("start_date")
176 {
177 return ImportEntityType::Event;
178 }
179 if columns.contains("project_type")
180 || (columns.contains("name") && !columns.contains("description"))
181 {
182 return ImportEntityType::Project;
183 }
184 ImportEntityType::Task
185 }
186
187 /// First non-empty value among the given (lowercased) candidate column names.
188 fn get_field(row: &HashMap<String, String>, names: &[&str]) -> Option<String> {
189 for name in names {
190 if let Some(value) = row.get(*name) {
191 let trimmed = value.trim();
192 if !trimmed.is_empty() {
193 return Some(trimmed.to_string());
194 }
195 }
196 }
197 None
198 }
199
200 fn parse_tasks(rows: &[HashMap<String, String>]) -> (Vec<ImportItem>, Vec<String>) {
201 let mut items = Vec::new();
202 let mut warnings = Vec::new();
203
204 for (idx, row) in rows.iter().enumerate() {
205 let description =
206 match get_field(row, &["description", "task", "title", "name", "subject"]) {
207 Some(d) => d,
208 None => {
209 warnings.push(format!("Row {}: missing description, skipped.", idx + 1));
210 continue;
211 }
212 };
213
214 let data = ImportTaskData {
215 description,
216 due: get_field(row, &["due", "due_date", "deadline", "date"]).and_then(|d| normalize_date(&d)),
217 priority: normalize_priority(get_field(row, &["priority", "pri", "importance"])),
218 status: normalize_task_status(get_field(row, &["status", "state"])),
219 project_name: get_field(row, &["project", "project_name", "category"]),
220 tags: Some(parse_tags(get_field(row, &["tags", "labels", "categories"]))),
221 notes: get_field(row, &["notes", "note", "comments", "body"]),
222 };
223
224 items.push(ImportItem {
225 source_index: idx + 1,
226 data: ImportItemData::Task(data),
227 has_errors: false,
228 errors: Vec::new(),
229 });
230 }
231
232 (items, warnings)
233 }
234
235 fn parse_projects(rows: &[HashMap<String, String>]) -> (Vec<ImportItem>, Vec<String>) {
236 let mut items = Vec::new();
237 let mut warnings = Vec::new();
238
239 for (idx, row) in rows.iter().enumerate() {
240 let name = match get_field(row, &["name", "project", "title"]) {
241 Some(n) => n,
242 None => {
243 warnings.push(format!("Row {}: missing name, skipped.", idx + 1));
244 continue;
245 }
246 };
247
248 let data = ImportProjectData {
249 name,
250 description: get_field(row, &["description", "desc", "notes"]),
251 project_type: normalize_project_type(get_field(row, &["type", "project_type", "category"])),
252 status: normalize_project_status(get_field(row, &["status", "state"])),
253 };
254
255 items.push(ImportItem {
256 source_index: idx + 1,
257 data: ImportItemData::Project(data),
258 has_errors: false,
259 errors: Vec::new(),
260 });
261 }
262
263 (items, warnings)
264 }
265
266 fn parse_events(rows: &[HashMap<String, String>]) -> (Vec<ImportItem>, Vec<String>) {
267 let mut items = Vec::new();
268 let mut warnings = Vec::new();
269
270 for (idx, row) in rows.iter().enumerate() {
271 let title = match get_field(row, &["title", "name", "event", "subject", "summary"]) {
272 Some(t) => t,
273 None => {
274 warnings.push(format!("Row {}: missing title, skipped.", idx + 1));
275 continue;
276 }
277 };
278
279 let start_raw = match get_field(row, &["start", "start_time", "start_date", "date", "when"]) {
280 Some(s) => s,
281 None => {
282 warnings.push(format!("Row {}: missing start time, skipped.", idx + 1));
283 continue;
284 }
285 };
286 // Normalize if recognizable, otherwise keep the raw value so the
287 // executor's parse_datetime gets a chance and reports a precise failure.
288 let start = normalize_date(&start_raw).unwrap_or(start_raw);
289
290 let data = ImportEventData {
291 title,
292 start,
293 end: get_field(row, &["end", "end_time", "end_date"]).and_then(|e| normalize_date(&e)),
294 location: get_field(row, &["location", "place", "venue", "where"]),
295 description: get_field(row, &["description", "notes", "body", "details"]),
296 project_name: get_field(row, &["project", "project_name", "category"]),
297 };
298
299 items.push(ImportItem {
300 source_index: idx + 1,
301 data: ImportItemData::Event(data),
302 has_errors: false,
303 errors: Vec::new(),
304 });
305 }
306
307 (items, warnings)
308 }
309
310 /// Maps loose priority spellings to High/Medium/Low. `None` input → `None`.
311 fn normalize_priority(value: Option<String>) -> Option<String> {
312 let value = value?;
313 Some(match value.to_lowercase().as_str() {
314 "high" | "1" | "h" | "urgent" | "critical" => "High",
315 "low" | "3" | "l" | "minor" => "Low",
316 _ => "Medium",
317 }
318 .to_string())
319 }
320
321 fn normalize_task_status(value: Option<String>) -> Option<String> {
322 let value = value?;
323 Some(match value.to_lowercase().as_str() {
324 "done" | "complete" | "completed" | "finished" | "closed" => "Done",
325 "in progress" | "inprogress" | "started" | "working" | "active" => "InProgress",
326 _ => "Pending",
327 }
328 .to_string())
329 }
330
331 fn normalize_project_type(value: Option<String>) -> Option<String> {
332 let value = value?;
333 Some(match value.to_lowercase().as_str() {
334 "job" | "work" | "employment" => "Job",
335 "side project" | "sideproject" | "personal" | "hobby" => "SideProject",
336 "company" | "business" | "startup" => "Company",
337 "essay" | "writing" => "Essay",
338 "article" | "blog" | "post" => "Article",
339 "painting" | "art" | "visual" => "Painting",
340 _ => "Other",
341 }
342 .to_string())
343 }
344
345 fn normalize_project_status(value: Option<String>) -> Option<String> {
346 let value = value?;
347 Some(match value.to_lowercase().as_str() {
348 "on hold" | "onhold" | "paused" | "waiting" => "OnHold",
349 "completed" | "done" | "finished" => "Completed",
350 "archived" | "inactive" | "closed" => "Archived",
351 _ => "Active",
352 }
353 .to_string())
354 }
355
356 /// Splits a delimited tag string (`,`, `;`, or `|`) into trimmed, non-empty tags.
357 fn parse_tags(value: Option<String>) -> Vec<String> {
358 match value {
359 None => Vec::new(),
360 Some(v) => v
361 .split([',', ';', '|'])
362 .map(|t| t.trim())
363 .filter(|t| !t.is_empty())
364 .map(|t| t.to_string())
365 .collect(),
366 }
367 }
368
369 /// Normalizes a date/datetime string to ISO form, trying common layouts.
370 /// Returns `None` if no layout matches (caller decides how to handle).
371 fn normalize_date(input: &str) -> Option<String> {
372 let input = input.trim();
373 if input.is_empty() {
374 return None;
375 }
376
377 const FORMATS: [&str; 10] = [
378 "%Y-%m-%d",
379 "%Y-%m-%dT%H:%M:%S",
380 "%Y-%m-%dT%H:%M:%SZ",
381 "%Y-%m-%dT%H:%M:%S%.fZ",
382 "%Y/%m/%d",
383 "%m/%d/%Y",
384 "%d/%m/%Y",
385 "%d-%m-%Y",
386 "%B %d, %Y",
387 "%b %d, %Y",
388 ];
389
390 for format in &FORMATS {
391 if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(input, format) {
392 return Some(dt.format("%Y-%m-%dT%H:%M:%S").to_string());
393 }
394 if let Ok(d) = chrono::NaiveDate::parse_from_str(input, format) {
395 return Some(d.format("%Y-%m-%d").to_string());
396 }
397 }
398 if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(input) {
399 return Some(dt.format("%Y-%m-%dT%H:%M:%S").to_string());
400 }
401 None
402 }
403
404 // ============ Import Executors ============
405
406 /// Imports parsed task items. Resolves project by name (case-insensitive),
407 /// parses priority and due date; per-item errors are accumulated.
408 async fn import_tasks(
409 state: &AppState,
410 items: &[&ImportItem],
411 projects: &[(String, String)],
412 ) -> Result<ImportExecuteResult, ApiError> {
413 let mut imported = 0;
414 let mut failed = 0;
415 let mut failures = Vec::new();
416
417 for item in items {
418 if let ImportItemData::Task(data) = &item.data {
419 let project_id = resolve_project(data.project_name.as_deref(), projects);
420
421 let priority = data
422 .priority
423 .as_ref()
424 .map(|p| match p.to_lowercase().as_str() {
425 "high" | "1" => Priority::High,
426 "low" | "3" => Priority::Low,
427 _ => Priority::Medium,
428 })
429 .unwrap_or(Priority::Medium);
430
431 let due = data.due.as_ref().and_then(|d| parse_datetime(d));
432
433 let mut builder = NewTaskBuilder::new(&data.description)
434 .priority(priority)
435 .tags(data.tags.clone().unwrap_or_default());
436 if let Some(d) = due {
437 builder = builder.due(d);
438 }
439 if let Some(pid) = project_id {
440 builder = builder.project_id(pid);
441 }
442
443 match state.tasks.create(DESKTOP_USER_ID, builder.build()).await {
444 Ok(_) => imported += 1,
445 Err(e) => {
446 failed += 1;
447 failures.push(ImportFailure {
448 source_index: item.source_index,
449 message: e.to_string(),
450 });
451 }
452 }
453 }
454 }
455
456 Ok(ImportExecuteResult {
457 imported_count: imported,
458 failed_count: failed,
459 skipped_count: items.len() - imported - failed,
460 failures,
461 })
462 }
463
464 /// Imports parsed project items. Type/status fall back to defaults on
465 /// unrecognized values rather than failing the import.
466 async fn import_projects(
467 state: &AppState,
468 items: &[&ImportItem],
469 ) -> Result<ImportExecuteResult, ApiError> {
470 let mut imported = 0;
471 let mut failed = 0;
472 let mut failures = Vec::new();
473
474 for item in items {
475 if let ImportItemData::Project(data) = &item.data {
476 let new_project = NewProject {
477 name: data.name.clone(),
478 description: data.description.clone().unwrap_or_default(),
479 project_type: data
480 .project_type
481 .as_ref()
482 .map(|t| goingson_core::ProjectType::from_str_or_default(t))
483 .unwrap_or_default(),
484 status: data
485 .status
486 .as_ref()
487 .map(|s| goingson_core::ProjectStatus::from_str_or_default(s))
488 .unwrap_or_default(),
489 };
490
491 match state.projects.create(DESKTOP_USER_ID, new_project).await {
492 Ok(_) => imported += 1,
493 Err(e) => {
494 failed += 1;
495 failures.push(ImportFailure {
496 source_index: item.source_index,
497 message: e.to_string(),
498 });
499 }
500 }
501 }
502 }
503
504 Ok(ImportExecuteResult {
505 imported_count: imported,
506 failed_count: failed,
507 skipped_count: items.len() - imported - failed,
508 failures,
509 })
510 }
511
512 /// Imports parsed event items. `start` is required — events without a parseable
513 /// start time are counted as failures.
514 async fn import_events(
515 state: &AppState,
516 items: &[&ImportItem],
517 projects: &[(String, String)],
518 ) -> Result<ImportExecuteResult, ApiError> {
519 let mut imported = 0;
520 let mut failed = 0;
521 let mut failures = Vec::new();
522
523 for item in items {
524 if let ImportItemData::Event(data) = &item.data {
525 let project_id = resolve_project(data.project_name.as_deref(), projects);
526
527 let start = match parse_datetime(&data.start) {
528 Some(dt) => dt,
529 None => {
530 failed += 1;
531 failures.push(ImportFailure {
532 source_index: item.source_index,
533 message: format!("Invalid start time: {}", data.start),
534 });
535 continue;
536 }
537 };
538 let end = data.end.as_ref().and_then(|e| parse_datetime(e));
539
540 let mut builder = NewEventBuilder::new(&data.title, start);
541 if let Some(e) = end {
542 builder = builder.end_time(e);
543 }
544 if let Some(ref loc) = data.location {
545 builder = builder.location(loc);
546 }
547 if let Some(ref desc) = data.description {
548 builder = builder.description(desc);
549 }
550 if let Some(pid) = project_id {
551 builder = builder.project_id(pid);
552 }
553
554 match state.events.create(DESKTOP_USER_ID, builder.build()).await {
555 Ok(_) => imported += 1,
556 Err(e) => {
557 failed += 1;
558 failures.push(ImportFailure {
559 source_index: item.source_index,
560 message: e.to_string(),
561 });
562 }
563 }
564 }
565 }
566
567 Ok(ImportExecuteResult {
568 imported_count: imported,
569 failed_count: failed,
570 skipped_count: items.len() - imported - failed,
571 failures,
572 })
573 }
574
575 /// Resolves a project name (case-insensitive) to its id.
576 fn resolve_project(name: Option<&str>, projects: &[(String, String)]) -> Option<ProjectId> {
577 let name = name?;
578 let name_lower = name.to_lowercase();
579 projects
580 .iter()
581 .find(|(_, n)| n.to_lowercase() == name_lower)
582 .and_then(|(id, _)| uuid::Uuid::parse_str(id).ok().map(ProjectId::from))
583 }
584
585 /// Parses a datetime with a three-format fallback (RFC 3339, ISO without tz
586 /// assumed UTC, date-only at midnight UTC). Returns `None` if none match.
587 fn parse_datetime(s: &str) -> Option<chrono::DateTime<chrono::Utc>> {
588 chrono::DateTime::parse_from_rfc3339(s)
589 .ok()
590 .map(|dt| dt.with_timezone(&chrono::Utc))
591 .or_else(|| {
592 chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S")
593 .ok()
594 .map(|dt| dt.and_utc())
595 })
596 .or_else(|| {
597 chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d")
598 .ok()
599 .map(|d| d.and_hms_opt(0, 0, 0).expect("midnight is valid").and_utc())
600 })
601 }
602
603 #[cfg(test)]
604 mod tests {
605 use super::*;
606
607 fn opts() -> ImportOptions {
608 ImportOptions {
609 has_header: true,
610 delimiter: None,
611 date_format: None,
612 extra: Default::default(),
613 }
614 }
615
616 #[test]
617 fn detects_event_from_start_column() {
618 let csv = "title,start\nStandup,2024-01-15\n";
619 let r = parse_csv_import(csv, &opts()).unwrap();
620 assert_eq!(r.entity_type, ImportEntityType::Event);
621 assert_eq!(r.items.len(), 1);
622 }
623
624 #[test]
625 fn detects_project_when_name_without_description() {
626 let csv = "name,status\nWebsite,active\n";
627 let r = parse_csv_import(csv, &opts()).unwrap();
628 assert_eq!(r.entity_type, ImportEntityType::Project);
629 }
630
631 #[test]
632 fn defaults_to_task_and_maps_fields() {
633 let csv = "description,priority,tags,project\nBuy milk,high,\"a, b\",Home\n";
634 let r = parse_csv_import(csv, &opts()).unwrap();
635 assert_eq!(r.entity_type, ImportEntityType::Task);
636 let ImportItemData::Task(t) = &r.items[0].data else {
637 panic!("expected task");
638 };
639 assert_eq!(t.description, "Buy milk");
640 assert_eq!(t.priority.as_deref(), Some("High"));
641 assert_eq!(t.project_name.as_deref(), Some("Home"));
642 assert_eq!(t.tags.as_ref().unwrap(), &vec!["a".to_string(), "b".to_string()]);
643 }
644
645 #[test]
646 fn case_insensitive_headers_and_bom_stripped() {
647 let csv = "\u{FEFF}Description,Due\nShip it,2024-03-01\n";
648 let r = parse_csv_import(csv, &opts()).unwrap();
649 let ImportItemData::Task(t) = &r.items[0].data else {
650 panic!("expected task");
651 };
652 assert_eq!(t.description, "Ship it");
653 assert_eq!(t.due.as_deref(), Some("2024-03-01"));
654 }
655
656 #[test]
657 fn rows_missing_required_field_become_warnings() {
658 // Second row has a priority but an empty description: a non-blank
659 // record missing its required field, which is warned and skipped.
660 // (Truly blank lines are dropped by the CSV reader, not warned.)
661 let csv = "description,priority\nReal task,high\n,low\n";
662 let r = parse_csv_import(csv, &opts()).unwrap();
663 assert_eq!(r.items.len(), 1);
664 assert_eq!(r.warnings.len(), 1);
665 }
666
667 #[test]
668 fn empty_file_yields_no_items() {
669 let r = parse_csv_import("", &opts()).unwrap();
670 assert!(r.items.is_empty());
671 }
672
673 #[test]
674 fn tsv_via_delimiter_option() {
675 let mut o = opts();
676 o.delimiter = Some('\t');
677 let csv = "description\tpriority\nTabbed\tlow\n";
678 let r = parse_csv_import(csv, &o).unwrap();
679 let ImportItemData::Task(t) = &r.items[0].data else {
680 panic!("expected task");
681 };
682 assert_eq!(t.description, "Tabbed");
683 assert_eq!(t.priority.as_deref(), Some("Low"));
684 }
685
686 #[test]
687 fn normalize_date_handles_us_format() {
688 assert_eq!(normalize_date("01/15/2024").as_deref(), Some("2024-01-15"));
689 assert_eq!(normalize_date("not a date"), None);
690 }
691 }
692