Add sync recovery commands for interrupted syncs

- Add wp mls recovery list to show resumable syncs
- Add wp mls recovery auto to auto-resume most recent failed sync
- Add wp mls recovery cleanup to mark stale syncs (>1hr) as failed
- Track last_next_link during incremental sync pagination
- Add get_resumable_syncs(), cleanup_stale_syncs(), auto_resume() methods

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Hanson.xyz Dev
2025-12-14 22:33:27 -06:00
parent 5e4ebfb99e
commit b62867d834
3 changed files with 254 additions and 0 deletions
@@ -36,6 +36,7 @@ class MLS_CLI {
WP_CLI::add_command('mls sync', array($instance, 'sync')); WP_CLI::add_command('mls sync', array($instance, 'sync'));
WP_CLI::add_command('mls stats', array($instance, 'stats')); WP_CLI::add_command('mls stats', array($instance, 'stats'));
WP_CLI::add_command('mls cache', array($instance, 'cache')); WP_CLI::add_command('mls cache', array($instance, 'cache'));
WP_CLI::add_command('mls recovery', array($instance, 'recovery'));
} }
/** /**
@@ -684,6 +685,144 @@ class MLS_CLI {
} }
} }
/**
* Manage sync recovery and resumption.
*
* ## OPTIONS
*
* <action>
* : Action: list, auto, cleanup
*
* [--verbose]
* : Show detailed output during resume
*
* [--quiet]
* : Suppress progress output
*
* ## EXAMPLES
*
* wp mls recovery list # Show resumable syncs
* wp mls recovery auto # Auto-resume most recent failed sync
* wp mls recovery auto --verbose # Auto-resume with detailed output
* wp mls recovery cleanup # Mark stale syncs as failed
*
* @subcommand recovery
*/
public function recovery($args, $assoc_args) {
$action = isset($args[0]) ? $args[0] : 'list';
$verbose = isset($assoc_args['verbose']);
$quiet = isset($assoc_args['quiet']);
$sync_engine = $this->plugin->get_sync_engine();
switch ($action) {
case 'list':
$resumable = $sync_engine->get_resumable_syncs();
if (empty($resumable)) {
WP_CLI::success('No resumable syncs found.');
break;
}
WP_CLI::line('');
WP_CLI::line('=== Resumable Syncs ===');
WP_CLI::line('');
foreach ($resumable as $sync) {
$status_color = $sync->status === 'failed' ? '%R' : '%Y';
WP_CLI::line(sprintf(
'ID: %d | Type: %s | Status: %s | Processed: %d',
$sync->id,
$sync->sync_type,
$sync->status,
$sync->records_processed
));
WP_CLI::line(sprintf(
' Started: %s | Updated: %s',
$sync->started_at,
$sync->updated_at
));
if ($sync->last_error) {
WP_CLI::warning(' Error: ' . $sync->last_error);
}
if ($sync->last_next_link) {
WP_CLI::line(' Has resume point: Yes');
}
WP_CLI::line('');
}
WP_CLI::line('To resume a specific sync: wp mls sync resume --id=<ID>');
WP_CLI::line('To auto-resume the most recent: wp mls recovery auto');
break;
case 'auto':
WP_CLI::line('Checking for resumable syncs...');
// Build progress callback
$progress_callback = null;
if (!$quiet) {
$progress_callback = function($event, $data = array()) use ($verbose) {
if ($verbose) {
$this->output_verbose_event($event, $data);
} else {
switch ($event) {
case 'property_created':
echo '.';
break;
case 'property_updated':
echo '#';
break;
case 'property_deleted':
echo 'x';
break;
case 'media_downloaded':
echo 'P';
break;
case 'media_skipped':
echo 'p';
break;
case 'media_error':
echo 'E';
break;
case 'page_complete':
echo '|';
break;
}
}
};
}
$result = $sync_engine->auto_resume($progress_callback);
if ($result === null) {
WP_CLI::success('No syncs to resume.');
break;
}
if (!$quiet && !$verbose) {
echo "\n";
}
$this->output_sync_result($result);
break;
case 'cleanup':
WP_CLI::line('Cleaning up stale syncs...');
$cleaned = $sync_engine->cleanup_stale_syncs();
if ($cleaned > 0) {
WP_CLI::success("Marked {$cleaned} stale sync(s) as failed.");
} else {
WP_CLI::success('No stale syncs found.');
}
break;
default:
WP_CLI::error("Unknown action: {$action}. Use 'list', 'auto', or 'cleanup'.");
}
}
/** /**
* Recursively delete a directory * Recursively delete a directory
*/ */
@@ -76,6 +76,11 @@ wp mls cache cleanup
wp mls cache missing # View failed media downloads wp mls cache missing # View failed media downloads
wp mls cache missing --limit=20 # View first 20 entries wp mls cache missing --limit=20 # View first 20 entries
wp mls cache missing --clear # Clear the log wp mls cache missing --clear # Clear the log
# Recovery commands
wp mls recovery list # Show resumable syncs
wp mls recovery auto # Auto-resume most recent failed sync
wp mls recovery cleanup # Mark stale (>1hr) syncs as failed
``` ```
### Progress Output ### Progress Output
@@ -100,6 +105,22 @@ Format: `[timestamp] listing_key | media_key | error | url`
Media downloads use exponential backoff (1s, 2s, 4s, 8s, 16s) for rate limit (429) and server errors (5xx). Media downloads use exponential backoff (1s, 2s, 4s, 8s, 16s) for rate limit (429) and server errors (5xx).
### Sync Recovery
The sync engine saves progress after each page, allowing interrupted syncs to resume:
1. **Automatic state tracking**: `last_next_link` saved after each API page
2. **Stale sync detection**: Syncs running >1 hour marked as failed
3. **Resume commands**:
- `wp mls sync resume --id=<ID>` - Resume specific sync
- `wp mls recovery auto` - Auto-resume most recent failed sync
- `wp mls recovery list` - View all resumable syncs
For cron jobs, consider adding recovery at the start:
```bash
wp mls recovery auto --quiet && wp mls sync incremental
```
### Public API Functions ### Public API Functions
Available for themes/plugins: Available for themes/plugins:
@@ -317,6 +317,17 @@ class MLS_Sync_Engine {
// Check for next page // Check for next page
if (isset($response['@odata.nextLink'])) { if (isset($response['@odata.nextLink'])) {
// Save progress for resume capability
if (!$dry_run) {
$this->update_sync_state(array(
'last_next_link' => $response['@odata.nextLink'],
'records_processed' => $this->stats['processed'],
'records_created' => $this->stats['created'],
'records_updated' => $this->stats['updated'],
'records_deleted' => $this->stats['deleted'],
));
}
$start_time = microtime(true); $start_time = microtime(true);
$this->emit_progress('api_request', array( $this->emit_progress('api_request', array(
'method' => 'GET', 'method' => 'GET',
@@ -809,4 +820,87 @@ class MLS_Sync_Engine {
'stats' => $this->db->get_stats(), 'stats' => $this->db->get_stats(),
); );
} }
/**
* Get resumable (failed or interrupted) syncs
*
* @return array List of resumable sync states
*/
public function get_resumable_syncs() {
global $wpdb;
// Find failed syncs that have a next_link (can be resumed)
// Also find "running" syncs older than 1 hour (likely interrupted)
$one_hour_ago = date('Y-m-d H:i:s', strtotime('-1 hour'));
return $wpdb->get_results($wpdb->prepare(
"SELECT * FROM {$this->db->sync_state_table()}
WHERE (status = 'failed' AND last_next_link IS NOT NULL)
OR (status = 'running' AND updated_at < %s)
ORDER BY updated_at DESC",
$one_hour_ago
));
}
/**
* Get the most recent resumable sync
*
* @return object|null Sync state or null
*/
public function get_latest_resumable() {
$resumable = $this->get_resumable_syncs();
return !empty($resumable) ? $resumable[0] : null;
}
/**
* Mark stale running syncs as failed
* Call this on startup to clean up interrupted syncs
*
* @param int $hours_threshold Hours after which running sync is considered stale
* @return int Number of syncs marked as failed
*/
public function cleanup_stale_syncs($hours_threshold = 1) {
global $wpdb;
$threshold_time = date('Y-m-d H:i:s', strtotime("-{$hours_threshold} hour"));
$updated = $wpdb->query($wpdb->prepare(
"UPDATE {$this->db->sync_state_table()}
SET status = 'failed', last_error = 'Sync interrupted (stale)'
WHERE status = 'running' AND updated_at < %s",
$threshold_time
));
if ($updated > 0) {
$this->logger->info('Cleaned up stale syncs', array('count' => $updated));
}
return $updated;
}
/**
* Auto-resume the most recent failed/interrupted sync
*
* @param callable|null $progress_callback Progress callback
* @return array|null Sync results or null if nothing to resume
*/
public function auto_resume($progress_callback = null) {
// First clean up any stale syncs
$this->cleanup_stale_syncs();
// Find the most recent resumable sync
$resumable = $this->get_latest_resumable();
if (!$resumable) {
return null;
}
$this->logger->info('Auto-resuming sync', array(
'sync_id' => $resumable->id,
'type' => $resumable->sync_type,
'processed' => $resumable->records_processed,
));
return $this->resume_sync($resumable->id, $progress_callback);
}
} }