// load_events.cc - Don Yang (uguu.org) // // 05/26/11 #include"load_events.h" #include #include #include #include #include"unicode.h" #include"util.h" using namespace std; // Partial snapshot of lines. This is generated after each frame is // processed to produce the set of animation instructions. class AllEvents::PartialSnapshot { public: // List of line contents for this snapshot. If not NULL, points at // a line inside the event list that will outlive this class. NULL // means line at this partial snapshot position is assumed to be // same as current snapshot. typedef deque Lines; PartialSnapshot(const CompiledEvent &base_event, const List &events, int *event_index); ~PartialSnapshot() {} inline int first_line() const { return first_line_; } inline int file_size() const { return file_size_; } inline const Lines &lines() const { return lines_; } inline const CompiledEvent &base_event() const { return base_event_; } private: // Replace snapshot line void Set(int index, const string *line); // Timing and cursor information const CompiledEvent &base_event_; // Snapshot contents int file_size_; int first_line_; Lines lines_; // No default constructor PartialSnapshot(); }; // Initialize partial snapshot and update event index AllEvents::PartialSnapshot::PartialSnapshot(const CompiledEvent &base_event, const List &events, int *event_index) : base_event_(base_event), file_size_(0), first_line_(-1) { const int start_frame = events[*event_index]->frame; for(; *event_index < static_cast(events.size()); ++*event_index) { const Event &next_event = *events[*event_index]; if( next_event.frame != start_frame || next_event.line < 0 ) break; Set(next_event.line, &(next_event.text)); file_size_ = next_event.size; while( first_line_ + static_cast(lines_.size()) > file_size_ ) lines_.pop_back(); } } // Replace snapshot line void AllEvents::PartialSnapshot::Set(int index, const string *line) { if( first_line_ < 0 ) { // Add line to empty snapshot assert(lines_.empty()); first_line_ = index; lines_.push_back(line); return; } if( first_line_ <= index ) { // Append line to snapshot while( first_line_ + static_cast(lines_.size()) <= index ) lines_.push_back(NULL); } else { // Prepend line to snapshot while( index < first_line_ ) { lines_.push_front(NULL); first_line_--; } } lines_[index - first_line_] = line; } ////////////////////////////////////////////////////////////////////// // Full snapshot of the entire file. This is updated continuously // through all frames. class AllEvents::FullSnapshot { public: FullSnapshot(CompiledEventList *events, OutputSlots *output_slots) : events_(events), output_slots_(output_slots) {} ~FullSnapshot() { DeletePtrElements(&lines_); } // Apply partial snapshot to full snapshot void Apply(const PartialSnapshot &snapshot); // Check cursor position and determine which type of cursor event to output. void PopulateCursorEvent(int row, int column, CompiledEvent *event); private: struct UnitLine { UnitLine(const string *text0, OutputSlots::iterator slot0) : text(text0), slot(slot0) {} // Text at current line const string *text; // Iterator to allocated slot OutputSlots::iterator slot; }; typedef list Lines; typedef vector NextLines; // Syntactic sugar inline int file_size() const { return static_cast(lines_.size()); } // Build partial snapshot with all NULL lines resolved void ResolveSameLines(const PartialSnapshot &snapshot, NextLines *next_lines) const; // Allocate new output slots, or return unused output slots OutputSlots::iterator AllocateLineBefore(OutputSlots::iterator position); OutputSlots::iterator AllocateLineAfter(OutputSlots::iterator position); // Create new compiled event, getting timestamp from partial snapshot static CompiledEvent *NewCompiledEvent(const PartialSnapshot &snapshot); // Update lines_ so that it contains the same number of lines as // the next snapshot. void CompareAndInsert(const PartialSnapshot &snapshot, const NextLines &next_lines); void CompareAndDelete(const PartialSnapshot &snapshot, const NextLines &next_lines); // Update changed lines void CompareAndUpdate(const PartialSnapshot &snapshot, const NextLines &next_lines); // Insert line before a particular position and update events_ void Insert(const PartialSnapshot &snapshot, Lines::iterator position, const string *new_line); // Delete a line at a particular position and update events_ void Delete(const PartialSnapshot &snapshot, Lines::iterator position); // Pre-allocated empty line const string empty_line_; // Output event list and slots CompiledEventList *events_; OutputSlots *output_slots_; // Pending snapshot of file, updated after each Apply() Lines lines_; // No default constructor FullSnapshot(); }; // Build partial snapshot with all NULL lines resolved void AllEvents::FullSnapshot::ResolveSameLines(const PartialSnapshot &snapshot, NextLines *next_lines) const { assert(!snapshot.lines().empty()); next_lines->reserve(snapshot.lines().size()); Lines::const_iterator p = SeekForward(lines_, snapshot.first_line()); for(PartialSnapshot::Lines::const_iterator q = snapshot.lines().begin(); q != snapshot.lines().end(); ++q) { if( p != lines_.end() ) { assert((*p)->text != NULL); next_lines->push_back(*q != NULL ? *q : (*p)->text); ++p; } else { next_lines->push_back(*q != NULL ? *q : &empty_line_); } } } // Allocate an output slot before a specified position AllEvents::OutputSlots::iterator AllEvents::FullSnapshot::AllocateLineBefore( OutputSlots::iterator position) { if( position != output_slots_->begin() ) { OutputSlots::iterator slot = position; --slot; if( !(slot->first) ) { slot->first = true; return slot; } } return output_slots_->insert(position, make_pair(true, 0)); } // Allocate an output slot after a specified position AllEvents::OutputSlots::iterator AllEvents::FullSnapshot::AllocateLineAfter( OutputSlots::iterator position) { if( position != output_slots_->end() ) { ++position; if( position != output_slots_->end() && !(position->first) ) { position->first = true; return position; } } return output_slots_->insert(position, make_pair(true, 0)); } // Create new compiled event AllEvents::CompiledEvent *AllEvents::FullSnapshot::NewCompiledEvent( const PartialSnapshot &snapshot) { CompiledEvent *event = new CompiledEvent; event->timestamp = snapshot.base_event().timestamp; event->frame = snapshot.base_event().frame; event->max_frame = snapshot.base_event().max_frame; return event; } // Apply partial snapshot to full snapshot void AllEvents::FullSnapshot::Apply(const PartialSnapshot &snapshot) { if( snapshot.lines().empty() ) return; // Resolve NULL entries in partial snapshot. Do this now before // inserting or removing any lines, otherwise it becomes a mess to // get the old line numbers. NextLines next_lines; ResolveSameLines(snapshot, &next_lines); // Apply changes so that the two snapshots end up with equal size if( file_size() < snapshot.file_size() ) CompareAndInsert(snapshot, next_lines); else if( file_size() > snapshot.file_size() ) CompareAndDelete(snapshot, next_lines); // Replace changed lines assert(file_size() == snapshot.file_size()); CompareAndUpdate(snapshot, next_lines); } // Insert lines until file size in current frame matches next frame. void AllEvents::FullSnapshot::CompareAndInsert(const PartialSnapshot &snapshot, const NextLines &next_lines) { // Insert gaps up to the first line while( file_size() < snapshot.first_line() ) Insert(snapshot, lines_.end(), &empty_line_); int lines_to_insert = snapshot.file_size() - file_size(); // Seek to first difference Lines::iterator p = SeekForward(&lines_, snapshot.first_line()); const int snapshot_size = static_cast(next_lines.size()); for(int q = 0; p != lines_.end() && q < snapshot_size;) { // If line at current frame matches line at next frame, we don't // want to insert a gap here. if( *((*p)->text) == *(next_lines[q]) ) { ++p; ++q; continue; } // Seek forward in next frame to look for a match. Assume that // P represents current lines and Q represents next lines, we // should be in a state like this: // [P-1] ? [P0][P1...] // [Q-1][Q0][Q1][Q2...] // i.e. if we insert a line gap before P0, we might get lucky and // all lines will match from there on. This is especially true // since inserts of a few lines are generally the common case. int d; bool matched = false; for(d = 1; d <= lines_to_insert && q + d < snapshot_size; d++) { if( *((*p)->text) == *(next_lines[q + d]) ) { matched = true; break; } } if( matched ) { lines_to_insert -= d; for(; d > 0; d--) Insert(snapshot, p, next_lines[q++]); } ++p; ++q; } // Insert blank lines at the end for(; lines_to_insert > 0; lines_to_insert--) Insert(snapshot, lines_.end(), &empty_line_); } // Remove lines until file size in current frame matches next frame void AllEvents::FullSnapshot::CompareAndDelete(const PartialSnapshot &snapshot, const NextLines &next_lines) { // Seek to first difference assert(snapshot.first_line() < file_size()); Lines::iterator p = SeekForward(&lines_, snapshot.first_line()); int lines_to_delete = file_size() - snapshot.file_size(); const int snapshot_size = static_cast(next_lines.size()); for(int q = 0; p != lines_.end() && q < snapshot_size;) { // Keep current line if it matches line at next frame if( *((*p)->text) == *(next_lines[q]) ) { ++p; ++q; continue; } // Seek forward in next frame to look for a match. Assume that // P represents current lines and Q represents next lines, we // should be in a state like this: // [P-1][P0][P1][P2...] // [Q-1] x [Q0][Q1...] // i.e. if we delete P0, we might get lucky and all lines match // from there on. int d; bool matched = false; Lines::const_iterator r = p; assert(r != lines_.end()); for(d = 1; d <= lines_to_delete; d++) { ++r; if( r == lines_.end() ) break; if( *((*r)->text) == *next_lines[q] ) { matched = true; break; } } if( matched ) { lines_to_delete -= d; for(; d > 0; d--) { Lines::iterator remove_position = p++; Delete(snapshot, remove_position); } } else { ++p; } ++q; } // Delete lines at the end assert(lines_to_delete <= file_size()); for(; lines_to_delete > 0; lines_to_delete--) { Lines::iterator last_line = lines_.end(); --last_line; Delete(snapshot, last_line); } } // Update changed lines void AllEvents::FullSnapshot::CompareAndUpdate(const PartialSnapshot &snapshot, const NextLines &next_lines) { assert(file_size() > snapshot.first_line()); // Compare each line Lines::iterator p = SeekForward(&lines_, snapshot.first_line()); assert(p != lines_.end()); for(NextLines::const_iterator q = next_lines.begin(); q != next_lines.end(); ++p, ++q) { assert((*p)->text != NULL); const string ¤t = *((*p)->text); const string &next = **q; if( current == next ) continue; CompiledEvent *new_event = NewCompiledEvent(snapshot); new_event->output = (*p)->slot; assert(new_event->output->first); // See if line at next frame can be constructed by appending // some text to current line. If so, insert an APPEND event, // otherwise insert a REPLACE event. // // If current line is empty, REPLACE event is always inserted. // This works better with OptimizeEvents later. if( !current.empty() && current.size() < next.size() && next.compare(0, current.size(), current) == 0 ) { new_event->type = CompiledEvent::APPEND; new_event->text = next.substr(current.size()); } else { new_event->type = CompiledEvent::REPLACE; new_event->text = next; } events_->push_back(new_event); (*p)->text = *q; } } // Insert line before a particular position and update events_ void AllEvents::FullSnapshot::Insert(const PartialSnapshot &snapshot, Lines::iterator position, const string *new_line) { // Find a slot to insert OutputSlots::iterator slot; if( position == lines_.end() ) { if( lines_.empty() ) slot = AllocateLineBefore(output_slots_->end()); else slot = AllocateLineAfter(lines_.back()->slot); } else { slot = AllocateLineBefore((*position)->slot); } // Insert line in current snapshot assert(new_line != NULL); lines_.insert(position, new UnitLine(new_line, slot)); // Record insert event CompiledEvent *new_event = NewCompiledEvent(snapshot); new_event->type = CompiledEvent::REPLACE; new_event->text = *new_line; new_event->output = slot; events_->push_back(new_event); } // Delete a line at a particular position and update events_ void AllEvents::FullSnapshot::Delete(const PartialSnapshot &snapshot, Lines::iterator position) { // Mark the slot as available assert(position != lines_.end()); assert((*position)->slot->first); (*position)->slot->first = false; // Record delete event CompiledEvent *new_event = NewCompiledEvent(snapshot); new_event->type = CompiledEvent::DELETE; new_event->output = (*position)->slot; events_->push_back(new_event); delete *position; lines_.erase(position); } // Apply cursor translation and determine cursor width void AllEvents::FullSnapshot::PopulateCursorEvent( int row, int column, CompiledEvent *event) { // Populate raw position event->row = row; event->column = column; // Check the row that the cursor is currently at Lines::iterator i = SeekForward(&lines_, row); if( i != lines_.end() ) { // Look for Unicode or tab characters up to the cursor. // // Normally we only need to do this for Unicode, since we can // actually predict the cursor position for tabs, but this // becomes a mess: do we expand the tabs to 8 spaces or // something else? Do we translate the cursor position? // Instead of doing either of these, we just use the slow cursor // feature. bool slow_cursor = false; const string &text = *((*i)->text); for(int j = 0; j < static_cast(text.size()) && j <= column; j++) { if( (static_cast(text[j]) & 0x80) != 0 || text[j] == '\t' ) { slow_cursor = true; break; } } if( slow_cursor ) { // Cursor was preceded by non-ASCII characters, or cursor is // on a non-ASCII character. Slow rendering mode required. event->type = CompiledEvent::SLOW_CURSOR; event->output = (*i)->slot; // Copy current line up to and including the cursor, pad with // extra half-width spaces if needed. event->text = text; while( static_cast(event->text.size()) <= column ) event->text.push_back(' '); // Erase all bytes after the cursor character string::iterator j = event->text.begin(); j += column; NextChar(&(event->text), &j); event->text.erase(j, event->text.end()); return; } } // Cursor is past the end of the file, or all characters up to and including // the cursor are ASCII. Fast cursor render mode can be used here. event->type = CompiledEvent::CURSOR; event->output = output_slots_->end(); } ////////////////////////////////////////////////////////////////////// AllEvents::AllEvents() : max_file_size_(0) {} AllEvents::~AllEvents() { DeletePtrElements(&events_); DeletePtrElements(&compiled_events_); } // Load input void AllEvents::Load(FILE *input) { assert(input != NULL); static const int kBufferSize = 256; char *read_buffer = new char[kBufferSize]; vector all_lines; Event sentinel_event; sentinel_event.row = sentinel_event.column = sentinel_event.frame = sentinel_event.timestamp = sentinel_event.line = sentinel_event.size = -1; const Event *last_event = events_.empty() ? &sentinel_event : events_.back(); // Parse input line by line Event *new_event = new Event; string line; int base_frame = 0; int base_timestamp = 0; int snapshot_line, snapshot_size; while( fgets(read_buffer, kBufferSize, input) != NULL ) { line.append(read_buffer); if( line.empty() || line[line.size() - 1] != '\n' ) continue; line.erase(line.size() - 1); if( !line.empty() && line[line.size() - 1] == '\r' ) line.erase(line.size() - 1); if( sscanf(line.c_str(), "Y%dX%dF%dT%d", &(new_event->row), &(new_event->column), &(new_event->frame), &(new_event->timestamp)) == 4 && new_event->row > 0 && new_event->column > 0 ) { // Cursor movement event. Convert cursor position from // 1-based to 0-based indices new_event->row--; new_event->column--; // Adjust timestamp for multi-session records. Do this before // filtering duplicates, otherwise the second duplicate record // after a timestamp rollover will not be dropped. new_event->frame += base_frame; new_event->timestamp += base_timestamp; if( new_event->frame < last_event->frame ) { base_frame += last_event->frame - new_event->frame + 1; base_timestamp += last_event->timestamp - new_event->timestamp + 1; new_event->frame = last_event->frame + 1; new_event->timestamp = last_event->timestamp + 1; } // Drop event if the cursor hasn't moved and timestamp hasn't changed if( new_event->row == last_event->row && new_event->column == last_event->column && new_event->frame == last_event->frame && new_event->timestamp == last_event->timestamp ) { line.clear(); continue; } // Add event last_event = new_event; events_.push_back(new_event); new_event = new Event; } else if( sscanf(line.c_str(), "L%dE%d=", &(snapshot_line), &(snapshot_size)) == 2 && snapshot_line > 0 && snapshot_line <= snapshot_size ) { // Convert line number from 1-based to 0-based snapshot_line--; // Line snapshot event const size_t separator = line.find('='); if( separator == string::npos || events_.empty() ) { line.clear(); continue; // Incomplete record } line.erase(0, separator + 1); // Drop event if we already have a snapshot for the current line if( snapshot_size == last_event->size && snapshot_line < static_cast(all_lines.size()) && all_lines[snapshot_line] != NULL && *(all_lines[snapshot_line]) == line ) { line.clear(); continue; } // Remember maximum number of lines in file if( max_file_size_ < snapshot_size ) max_file_size_ = snapshot_size; // Copy cursor position from most recent event new_event->row = last_event->row; new_event->column = last_event->column; new_event->frame = last_event->frame; new_event->timestamp = last_event->timestamp; new_event->line = snapshot_line; new_event->size = snapshot_size; new_event->text.swap(line); // Remember line for delta encoding all_lines.resize(snapshot_size); while( snapshot_line >= static_cast(all_lines.size()) ) all_lines.push_back(NULL); all_lines[snapshot_line] = &(new_event->text); last_event = new_event; events_.push_back(new_event); new_event = new Event; } line.clear(); } delete new_event; delete[] read_buffer; } // Reduce idleness void AllEvents::CompressIdleTime(int max_idle_seconds) { assert(max_idle_seconds >= 1); if( events_.empty() ) return; // Set initial adjustment time to remove any idleness up to the first event int adjust_time = (*(events_.begin()))->timestamp; int last_timestamp = 0; for(List::iterator i = events_.begin(); i != events_.end(); ++i) { Event *event = *i; event->timestamp -= adjust_time; if( event->timestamp - last_timestamp > max_idle_seconds ) { adjust_time += event->timestamp - last_timestamp - max_idle_seconds; event->timestamp = last_timestamp + max_idle_seconds; } last_timestamp = event->timestamp; } } // Compile all events void AllEvents::CompileEvents() { assert(compiled_events_.empty()); assert(output_slots_.empty()); // (frame count numerator, frame count denominator) vector > frames; GetRelativeFrames(&frames); // Snapshot of current file FullSnapshot snapshot(&compiled_events_, &output_slots_); // Process raw events CompiledEvent *event_buffer = new CompiledEvent; for(int i = 0; i < static_cast(events_.size());) { // Initialize timestamps const Event &raw_event = event(i); event_buffer->timestamp = raw_event.timestamp; event_buffer->frame = frames[i].first; event_buffer->max_frame = frames[i].second; if( raw_event.line < 0 ) { // Cursor event snapshot.PopulateCursorEvent(raw_event.row, raw_event.column, event_buffer); compiled_events_.push_back(event_buffer); event_buffer = new CompiledEvent; i++; continue; } // Build partial snapshot of the new file up to the next frame const PartialSnapshot new_snapshot(*event_buffer, events_, &i); snapshot.Apply(new_snapshot); } delete event_buffer; // Assign serial numbers to output slots int slot_number = 0; for(OutputSlots::iterator i = output_slots_.begin(); i != output_slots_.end(); ++i) { i->second = slot_number++; } // Remove redundant entries OptimizeEvents(); } // Compute same second frame numerator/denominator pairs void AllEvents::GetRelativeFrames(vector > *frames) const { assert(frames != NULL); frames->reserve(events_.size()); for(int i = 0; i < static_cast(events_.size());) { // Find starting frame count at next timestamp const int start_timestamp = event(i).timestamp; int j = i + 1; for(; j < static_cast(events_.size()); j++) { if( event(j).timestamp != start_timestamp ) break; } // Use starting frame count at next second if available, // otherwise use max frame count from current second. const int start_frame = event(i).frame; const int frame_count = (j < static_cast(events_.size())) ? event(j).frame - start_frame : event(j - 1).frame + 1 - start_frame; for(; i < j; i++) { frames->push_back(make_pair(event(i).frame - start_frame, frame_count)); } } } // Remove redundant entries void AllEvents::OptimizeEvents() { // (slot, change==append) map changed_lines; int last_timestamp = -1; int last_frame = -1; bool duplicate_cursor = true; for(CompiledEventList::iterator i = compiled_events_.end(); i != compiled_events_.begin();) { --i; CompiledEvent *event = *i; // Check if we have moved on to a different frame if( event->timestamp != last_timestamp || event->frame != last_frame ) { last_timestamp = event->timestamp; last_frame = event->frame; changed_lines.clear(); duplicate_cursor = false; } // Drop earlier cursor events from the same frame if( event->type == CompiledEvent::CURSOR || event->type == CompiledEvent::SLOW_CURSOR ) { if( duplicate_cursor ) { CompiledEventList::iterator x = i++; compiled_events_.erase(x); delete event; } else { duplicate_cursor = true; } continue; } // Optimize away edits on the same slot. These are artefacts // due to how we apply each snapshots: the first insert/delete // phase may add extra lines, and the latter update phase will // operate on the same lines. This code will drop the redundant // entries from the first phase. const int slot = event->output->second; if( event->type == CompiledEvent::REPLACE || event->type == CompiledEvent::DELETE ) { pair::iterator, bool> p = changed_lines.insert(make_pair(slot, false)); if( p.second ) continue; if( p.first->second ) { // This REPLACE or DELETE event precedes an APPEND event. // We can't drop this event because it will invalidate the // APPEND, but we will set the flag here so that edit // events earlier than this one are dropped. p.first->second = false; continue; } } else { assert(event->type = CompiledEvent::APPEND); pair::iterator, bool> p = changed_lines.insert(make_pair(slot, true)); if( p.second ) continue; if( p.first->second ) { // This APPEND event precedes another APPEND event, // neither event can be dropped. continue; } } // Drop redundant event CompiledEventList::iterator x = i++; compiled_events_.erase(x); delete event; } }