From d3fb69cf129fe70c932a5d82fdd1bcc613544b5b Mon Sep 17 00:00:00 2001 From: Joris Date: Sun, 27 Nov 2022 15:38:39 +0100 Subject: Speed up deck synchronization Apply changes for what has been inserted, updated, removed. Also use transactions to speed up multi-writing. --- src/db/mod.rs | 116 +++++++++++++++------------ src/db/sql/3-drop-deck-read.sql | 1 + src/deck.rs | 12 +-- src/gui/message.rs | 2 +- src/gui/mod.rs | 30 +------ src/gui/question.rs | 10 ++- src/main.rs | 20 ++--- src/model/card.rs | 9 --- src/model/entry.rs | 5 -- src/model/mod.rs | 30 ++++++- src/sync.rs | 173 ++++++++++++++++++++++++++++++++++++++++ src/util/time.rs | 18 ----- 12 files changed, 291 insertions(+), 135 deletions(-) create mode 100644 src/db/sql/3-drop-deck-read.sql delete mode 100644 src/model/card.rs delete mode 100644 src/model/entry.rs create mode 100644 src/sync.rs diff --git a/src/db/mod.rs b/src/db/mod.rs index c2749dc..434d74a 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,81 +1,99 @@ -use crate::{ - model::{card::Card, entry::Entry}, - space_repetition, - util::serialization, -}; use anyhow::Result; use rusqlite::{params, Connection}; use rusqlite_migration::{Migrations, M}; +use crate::model::DbEntry; use crate::util::time; +use crate::{ + model::{Card, Question}, + space_repetition, + util::serialization, +}; pub fn init(database: String) -> Result { let mut conn = Connection::open(database)?; let migrations = Migrations::new(vec![ M::up(include_str!("sql/1-init.sql")), M::up(include_str!("sql/2-primary-key-question-responses.sql")), + M::up(include_str!("sql/3-drop-deck-read.sql")), ]); migrations.to_latest(&mut conn)?; Ok(conn) } -pub fn last_deck_read(conn: &Connection) -> Option { - let mut stmt = conn - .prepare("SELECT deck_read FROM cards ORDER BY deck_read DESC LIMIT 1") - .ok()?; - - let mut rows = stmt.query([]).ok()?; - let row = rows.next().ok()??; - row.get(0).ok()? +pub fn all(conn: &Connection) -> Result> { + let mut stmt = conn.prepare("SELECT question, responses, deleted FROM cards")?; + + let res: Result, rusqlite::Error> = stmt + .query_map([], |row| { + let responses: String = row.get(1)?; + Ok(DbEntry { + question: row.get(0)?, + responses: serialization::line_to_words(&responses), + deleted: row.get(2)?, + }) + })? + .collect(); + + Ok(res?) } -/// Synchronize the DB with the deck: -/// -/// - insert new cards, -/// - keep existing cards, -/// - hide unused cards (keep state in case the card is added back afterward). -pub fn synchronize(conn: &Connection, entries: Vec) -> Result<()> { +pub fn insert(conn: &mut Connection, questions: &Vec) -> Result<()> { let now = time::seconds_since_unix_epoch()?; - let state = serde_json::to_string(&space_repetition::init())?; - for entry in entries { - let concat_1 = serialization::words_to_line(&entry.part_1); - let concat_2 = serialization::words_to_line(&entry.part_2); - - for w in entry.part_1.iter() { - insert(conn, now, w, &concat_2, &state)?; - } - - for w in entry.part_2.iter() { - insert(conn, now, w, &concat_1, &state)?; - } + let tx = conn.transaction()?; + for Question { + question, + responses, + } in questions + { + let responses = serialization::words_to_line(responses); + tx.execute( + " + INSERT INTO cards (question, responses, state, created, ready) + VALUES (?, ?, ?, ?, ?) + ", + params![question, responses, state, now, now], + )?; } - - delete_read_before(conn, now)?; - + tx.commit()?; Ok(()) } -fn insert(conn: &Connection, now: u64, question: &str, responses: &str, state: &str) -> Result<()> { - conn.execute( - " - INSERT INTO cards (question, responses, state, created, deck_read, ready) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT (question, responses) DO UPDATE SET deck_read = ?, deleted = null - ", - params![question, responses, state, now, now, now, now], - )?; +pub fn delete(conn: &mut Connection, questions: &Vec) -> Result<()> { + let now = time::seconds_since_unix_epoch()?; + let tx = conn.transaction()?; + for Question { + question, + responses, + } in questions + { + let responses = serialization::words_to_line(responses); + tx.execute( + "UPDATE cards SET deleted = ? WHERE question = ? AND responses = ?", + params![now, question, responses], + )?; + } + tx.commit()?; Ok(()) } -fn delete_read_before(conn: &Connection, t: u64) -> Result<()> { - conn.execute( - "UPDATE cards SET deleted = ? WHERE deck_read < ?", - params![t, t], - )?; - +pub fn undelete(conn: &mut Connection, questions: &Vec) -> Result<()> { + let tx = conn.transaction()?; + for Question { + question, + responses, + } in questions + { + let responses = serialization::words_to_line(responses); + tx.execute( + "UPDATE cards SET deleted = NULL WHERE question = ? AND responses = ?", + params![question, responses], + )?; + } + tx.commit()?; Ok(()) } diff --git a/src/db/sql/3-drop-deck-read.sql b/src/db/sql/3-drop-deck-read.sql new file mode 100644 index 0000000..1ca23d1 --- /dev/null +++ b/src/db/sql/3-drop-deck-read.sql @@ -0,0 +1 @@ +ALTER TABLE cards DROP COLUMN deck_read; diff --git a/src/deck.rs b/src/deck.rs index d23529f..0c302e1 100644 --- a/src/deck.rs +++ b/src/deck.rs @@ -1,4 +1,4 @@ -use crate::{model::entry::Entry, util::serialization}; +use crate::{model::Line, util::serialization}; use anyhow::{Error, Result}; use std::fmt; use std::fs::File; @@ -23,10 +23,10 @@ impl std::error::Error for ParseError { } } -pub fn read(deck: &str) -> Result> { +pub fn read(deck: &str) -> Result> { let file = File::open(deck)?; let reader = BufReader::new(file); - let mut entries: Vec = Vec::new(); + let mut entries: Vec = Vec::new(); for (index, line) in reader.lines().enumerate() { let line = line?; @@ -57,9 +57,9 @@ pub fn read(deck: &str) -> Result> { .to_string(), })); } else { - entries.push(Entry { - part_1: serialization::line_to_words(&t1.to_string()), - part_2: serialization::line_to_words(&t2.to_string()), + entries.push(Line { + part_1: serialization::line_to_words(t1), + part_2: serialization::line_to_words(t2), }) } } diff --git a/src/gui/message.rs b/src/gui/message.rs index 1ae10a5..29b5d8a 100644 --- a/src/gui/message.rs +++ b/src/gui/message.rs @@ -27,7 +27,7 @@ pub fn show( let d1 = util::title(title); f.render_widget(d1, chunks[0]); - let message = Paragraph::new(util::center_vertically(chunks[1], &message.to_string())) + let message = Paragraph::new(util::center_vertically(chunks[1], message)) .alignment(Alignment::Center); f.render_widget(message, chunks[1]); })?; diff --git a/src/gui/mod.rs b/src/gui/mod.rs index 858b30d..358e4b5 100644 --- a/src/gui/mod.rs +++ b/src/gui/mod.rs @@ -2,12 +2,10 @@ pub mod message; pub mod question; pub mod util; -use crate::deck; -use crate::util::time; -use crate::{db, space_repetition, util::event::Events}; +use crate::{db, space_repetition, util::event::Events, util::time}; use anyhow::Result; use rusqlite::Connection; -use std::{fs, io, time::Duration}; +use std::io; use termion::{raw::IntoRawMode, raw::RawTerminal, screen::AlternateScreen}; use tui::{backend::TermionBackend, Terminal}; @@ -20,28 +18,6 @@ pub fn terminal() -> Result { Ok(Terminal::new(backend)?) } -pub fn synchronize( - conn: &Connection, - term: &mut Term, - events: &Events, - deck_path: &str, - deck_name: &str, -) -> Result<()> { - let last_modified = time::seconds_since_unix_epoch_of(fs::metadata(deck_path)?.modified()?)?; - let last_deck_read = db::last_deck_read(conn); - let must_synchronize = last_deck_read.map(|r| r < last_modified).unwrap_or(true); - - if must_synchronize { - let _ = message::show(term, events, deck_name, "Synchronization du deck…", false); - time::wait_at_least( - || db::synchronize(conn, deck::read(deck_path)?), - Duration::from_secs(1), - )?; - } - - Ok(()) -} - pub fn start(conn: &Connection, term: &mut Term, events: &Events, deck_name: &str) -> Result<()> { let mut answers = 0; @@ -66,7 +42,7 @@ pub fn start(conn: &Connection, term: &mut Term, events: &Events, deck_name: &st Some(ready) => { let duration = time::pp_duration(ready - now); format!("Prochaine carte disponible dans {duration}.") - }, + } None => "Aucune carte n’est disponible. Votre deck est-il vide ?".to_string(), }; let _ = message::show(term, events, &title, &message, true); diff --git a/src/gui/question.rs b/src/gui/question.rs index 71c3ea2..2aa6e65 100644 --- a/src/gui/question.rs +++ b/src/gui/question.rs @@ -1,6 +1,6 @@ use crate::{ gui::util, - model::{card::Card, difficulty, difficulty::Difficulty}, + model::{difficulty, difficulty::Difficulty, Card}, util::event::{Event, Events}, util::serialization, }; @@ -150,7 +150,9 @@ pub fn ask( Key::Char(c) => { state.input.push(c); if is_correct(&state.input, &card.responses) { - state.answer = Answer::Difficulty { difficulty: Difficulty::Good } + state.answer = Answer::Difficulty { + difficulty: Difficulty::Good, + } } } Key::Backspace => { @@ -161,12 +163,12 @@ pub fn ask( } Key::Ctrl('w') => { let mut words = state.input.split_whitespace().collect::>(); - if words.len() > 0 { + if !words.is_empty() { words.truncate(words.len() - 1); state.input = format!( "{}{}", words.join(" "), - if words.len() > 0 { " " } else { "" } + if !words.is_empty() { " " } else { "" } ); } } diff --git a/src/main.rs b/src/main.rs index 4ca3822..a791f29 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,11 +3,11 @@ mod deck; mod gui; mod model; mod space_repetition; +mod sync; mod util; use crate::util::event::Events; use anyhow::Result; -use rusqlite::Connection; use std::path::PathBuf; use structopt::StructOpt; @@ -20,11 +20,14 @@ struct Opt { fn main() -> Result<()> { let deck_path = Opt::from_args().deck; - let conn = db::init(db_path(&deck_path))?; + let mut conn = db::init(db_path(&deck_path))?; let deck_name = deck::pp_from_path(&deck_path).unwrap_or_else(|| "Deck".to_string()); + + sync::run(&mut conn, &deck_path)?; + let mut term = gui::terminal()?; let events = Events::new(); - match run_tui(conn, &deck_path, &deck_name, &mut term, &events) { + match gui::start(&conn, &mut term, &events, &deck_name) { Ok(()) => Ok(()), Err(msg) => { // Show errors in TUI, otherwise they are hidden @@ -34,17 +37,6 @@ fn main() -> Result<()> { } } -fn run_tui( - conn: Connection, - deck_path: &str, - deck_name: &str, - term: &mut gui::Term, - events: &Events, -) -> Result<()> { - gui::synchronize(&conn, term, &events, &deck_path, &deck_name)?; - gui::start(&conn, term, &events, &deck_name) -} - fn db_path(deck_path: &str) -> String { let mut path = PathBuf::from(deck_path); path.set_extension("db"); diff --git a/src/model/card.rs b/src/model/card.rs deleted file mode 100644 index 811f877..0000000 --- a/src/model/card.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::space_repetition; - -#[derive(Debug)] -pub struct Card { - pub question: String, - pub responses: Vec, - pub state: space_repetition::State, - pub ready: u64, -} diff --git a/src/model/entry.rs b/src/model/entry.rs deleted file mode 100644 index 769b38c..0000000 --- a/src/model/entry.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Debug, Clone)] -pub struct Entry { - pub part_1: Vec, - pub part_2: Vec, -} diff --git a/src/model/mod.rs b/src/model/mod.rs index 185f401..2dc1ab5 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1,3 +1,29 @@ -pub mod card; +use crate::space_repetition; + pub mod difficulty; -pub mod entry; + +#[derive(Debug, Clone)] +pub struct Line { + pub part_1: Vec, + pub part_2: Vec, +} + +pub struct DbEntry { + pub question: String, + pub responses: Vec, + pub deleted: Option, +} + +#[derive(Debug)] +pub struct Card { + pub question: String, + pub responses: Vec, + pub state: space_repetition::State, + pub ready: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Question { + pub question: String, + pub responses: Vec, +} diff --git a/src/sync.rs b/src/sync.rs new file mode 100644 index 0000000..3911d55 --- /dev/null +++ b/src/sync.rs @@ -0,0 +1,173 @@ +use crate::{ + db, deck, + model::{DbEntry, Line, Question}, +}; +use anyhow::Result; +use rusqlite::Connection; +use std::collections::HashSet; + +pub fn run(conn: &mut Connection, deck_path: &str) -> Result<()> { + let db_entries = db::all(conn)?; + let lines = deck::read(deck_path)?; + let Diff { + new, + deleted, + undeleted, + } = diff(db_entries, lines); + + db::insert(conn, &new)?; + db::delete(conn, &deleted)?; + db::undelete(conn, &undeleted)?; + + Ok(()) +} + +struct Diff { + pub new: Vec, + pub deleted: Vec, + pub undeleted: Vec, +} + +fn diff(db_entries: Vec, lines: Vec) -> Diff { + let mut file_questions: HashSet = HashSet::new(); + let mut db_questions_not_deleted: HashSet = HashSet::new(); + let mut db_questions_deleted: HashSet = HashSet::new(); + + for Line { part_1, part_2 } in lines { + for question in part_1.clone() { + let mut responses = part_2.clone(); + responses.sort(); + file_questions.insert(Question { + question, + responses, + }); + } + for question in part_2 { + let mut responses = part_1.clone(); + responses.sort(); + file_questions.insert(Question { + question, + responses, + }); + } + } + + for DbEntry { + question, + mut responses, + deleted, + } in db_entries + { + responses.sort(); + if deleted.is_some() { + db_questions_deleted.insert(Question { + question, + responses, + }); + } else { + db_questions_not_deleted.insert(Question { + question, + responses, + }); + } + } + + let new = file_questions + .difference(&db_questions_not_deleted) + .cloned() + .collect::>() + .difference(&db_questions_deleted) + .cloned() + .collect(); + + let deleted = db_questions_not_deleted + .difference(&file_questions) + .cloned() + .collect(); + + let undeleted = file_questions + .intersection(&db_questions_deleted) + .cloned() + .collect(); + + Diff { + new, + deleted, + undeleted, + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn sync() { + let db_entries = vec![ + DbEntry { + question: "A".to_string(), + responses: vec!["A".to_string()], + deleted: None, + }, + DbEntry { + question: "B".to_string(), + responses: vec!["B".to_string()], + deleted: None, + }, + DbEntry { + question: "C".to_string(), + responses: vec!["C".to_string()], + deleted: Some(0), + }, + DbEntry { + question: "D".to_string(), + responses: vec!["D".to_string()], + deleted: Some(0), + }, + ]; + + let lines = vec![ + Line { + part_1: vec!["A".to_string()], + part_2: vec!["A".to_string()], + }, + Line { + part_1: vec!["C".to_string()], + part_2: vec!["C".to_string()], + }, + Line { + part_1: vec!["E".to_string()], + part_2: vec!["E".to_string()], + }, + ]; + + let Res { + new, + deleted, + undeleted, + } = sync(db_entries, lines); + + assert_eq!( + new, + vec!(Question { + question: "E".to_string(), + responses: vec!("E".to_string()) + }) + ); + assert_eq!( + deleted, + vec!(Question { + question: "B".to_string(), + responses: vec!("B".to_string()) + }) + ); + assert_eq!( + undeleted, + vec!(Question { + question: "C".to_string(), + responses: vec!("C".to_string()) + }) + ); + } +} diff --git a/src/util/time.rs b/src/util/time.rs index 679d1b4..e4bf75c 100644 --- a/src/util/time.rs +++ b/src/util/time.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use std::thread; use std::time::SystemTime; pub fn seconds_since_unix_epoch() -> Result { @@ -36,20 +35,3 @@ fn plural(n: u64, str: &str) -> String { format!("{n} {str}s") } } - -/// Call the function, then sleep if necessary. -/// -/// Calling this will at least take the duration asked for in parameters. -pub fn wait_at_least(f: F, d: std::time::Duration) -> Result<()> -where - F: Fn() -> Result<()>, -{ - let t1 = SystemTime::now(); - f()?; - let t2 = SystemTime::now(); - let elapsed = t2.duration_since(t1)?; - if elapsed < d { - thread::sleep(d - elapsed); - } - Ok(()) -} -- cgit v1.2.3