Add sync recovery commands for interrupted syncs
- Add wp mls recovery list to show resumable syncs - Add wp mls recovery auto to auto-resume most recent failed sync - Add wp mls recovery cleanup to mark stale syncs (>1hr) as failed - Track last_next_link during incremental sync pagination - Add get_resumable_syncs(), cleanup_stale_syncs(), auto_resume() methods 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Hanson.xyz Dev
@@ -36,6 +36,7 @@ class MLS_CLI {
|
||||
WP_CLI::add_command('mls sync', array($instance, 'sync'));
|
||||
WP_CLI::add_command('mls stats', array($instance, 'stats'));
|
||||
WP_CLI::add_command('mls cache', array($instance, 'cache'));
|
||||
WP_CLI::add_command('mls recovery', array($instance, 'recovery'));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -684,6 +685,144 @@ class MLS_CLI {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Manage sync recovery and resumption.
|
||||
*
|
||||
* ## OPTIONS
|
||||
*
|
||||
* <action>
|
||||
* : Action: list, auto, cleanup
|
||||
*
|
||||
* [--verbose]
|
||||
* : Show detailed output during resume
|
||||
*
|
||||
* [--quiet]
|
||||
* : Suppress progress output
|
||||
*
|
||||
* ## EXAMPLES
|
||||
*
|
||||
* wp mls recovery list # Show resumable syncs
|
||||
* wp mls recovery auto # Auto-resume most recent failed sync
|
||||
* wp mls recovery auto --verbose # Auto-resume with detailed output
|
||||
* wp mls recovery cleanup # Mark stale syncs as failed
|
||||
*
|
||||
* @subcommand recovery
|
||||
*/
|
||||
public function recovery($args, $assoc_args) {
|
||||
$action = isset($args[0]) ? $args[0] : 'list';
|
||||
$verbose = isset($assoc_args['verbose']);
|
||||
$quiet = isset($assoc_args['quiet']);
|
||||
|
||||
$sync_engine = $this->plugin->get_sync_engine();
|
||||
|
||||
switch ($action) {
|
||||
case 'list':
|
||||
$resumable = $sync_engine->get_resumable_syncs();
|
||||
|
||||
if (empty($resumable)) {
|
||||
WP_CLI::success('No resumable syncs found.');
|
||||
break;
|
||||
}
|
||||
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line('=== Resumable Syncs ===');
|
||||
WP_CLI::line('');
|
||||
|
||||
foreach ($resumable as $sync) {
|
||||
$status_color = $sync->status === 'failed' ? '%R' : '%Y';
|
||||
WP_CLI::line(sprintf(
|
||||
'ID: %d | Type: %s | Status: %s | Processed: %d',
|
||||
$sync->id,
|
||||
$sync->sync_type,
|
||||
$sync->status,
|
||||
$sync->records_processed
|
||||
));
|
||||
WP_CLI::line(sprintf(
|
||||
' Started: %s | Updated: %s',
|
||||
$sync->started_at,
|
||||
$sync->updated_at
|
||||
));
|
||||
if ($sync->last_error) {
|
||||
WP_CLI::warning(' Error: ' . $sync->last_error);
|
||||
}
|
||||
if ($sync->last_next_link) {
|
||||
WP_CLI::line(' Has resume point: Yes');
|
||||
}
|
||||
WP_CLI::line('');
|
||||
}
|
||||
|
||||
WP_CLI::line('To resume a specific sync: wp mls sync resume --id=<ID>');
|
||||
WP_CLI::line('To auto-resume the most recent: wp mls recovery auto');
|
||||
break;
|
||||
|
||||
case 'auto':
|
||||
WP_CLI::line('Checking for resumable syncs...');
|
||||
|
||||
// Build progress callback
|
||||
$progress_callback = null;
|
||||
if (!$quiet) {
|
||||
$progress_callback = function($event, $data = array()) use ($verbose) {
|
||||
if ($verbose) {
|
||||
$this->output_verbose_event($event, $data);
|
||||
} else {
|
||||
switch ($event) {
|
||||
case 'property_created':
|
||||
echo '.';
|
||||
break;
|
||||
case 'property_updated':
|
||||
echo '#';
|
||||
break;
|
||||
case 'property_deleted':
|
||||
echo 'x';
|
||||
break;
|
||||
case 'media_downloaded':
|
||||
echo 'P';
|
||||
break;
|
||||
case 'media_skipped':
|
||||
echo 'p';
|
||||
break;
|
||||
case 'media_error':
|
||||
echo 'E';
|
||||
break;
|
||||
case 'page_complete':
|
||||
echo '|';
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
$result = $sync_engine->auto_resume($progress_callback);
|
||||
|
||||
if ($result === null) {
|
||||
WP_CLI::success('No syncs to resume.');
|
||||
break;
|
||||
}
|
||||
|
||||
if (!$quiet && !$verbose) {
|
||||
echo "\n";
|
||||
}
|
||||
|
||||
$this->output_sync_result($result);
|
||||
break;
|
||||
|
||||
case 'cleanup':
|
||||
WP_CLI::line('Cleaning up stale syncs...');
|
||||
|
||||
$cleaned = $sync_engine->cleanup_stale_syncs();
|
||||
|
||||
if ($cleaned > 0) {
|
||||
WP_CLI::success("Marked {$cleaned} stale sync(s) as failed.");
|
||||
} else {
|
||||
WP_CLI::success('No stale syncs found.');
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
WP_CLI::error("Unknown action: {$action}. Use 'list', 'auto', or 'cleanup'.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively delete a directory
|
||||
*/
|
||||
|
||||
@@ -76,6 +76,11 @@ wp mls cache cleanup
|
||||
wp mls cache missing # View failed media downloads
|
||||
wp mls cache missing --limit=20 # View first 20 entries
|
||||
wp mls cache missing --clear # Clear the log
|
||||
|
||||
# Recovery commands
|
||||
wp mls recovery list # Show resumable syncs
|
||||
wp mls recovery auto # Auto-resume most recent failed sync
|
||||
wp mls recovery cleanup # Mark stale (>1hr) syncs as failed
|
||||
```
|
||||
|
||||
### Progress Output
|
||||
@@ -100,6 +105,22 @@ Format: `[timestamp] listing_key | media_key | error | url`
|
||||
|
||||
Media downloads use exponential backoff (1s, 2s, 4s, 8s, 16s) for rate limit (429) and server errors (5xx).
|
||||
|
||||
### Sync Recovery
|
||||
|
||||
The sync engine saves progress after each page, allowing interrupted syncs to resume:
|
||||
|
||||
1. **Automatic state tracking**: `last_next_link` saved after each API page
|
||||
2. **Stale sync detection**: Syncs running >1 hour marked as failed
|
||||
3. **Resume commands**:
|
||||
- `wp mls sync resume --id=<ID>` - Resume specific sync
|
||||
- `wp mls recovery auto` - Auto-resume most recent failed sync
|
||||
- `wp mls recovery list` - View all resumable syncs
|
||||
|
||||
For cron jobs, consider adding recovery at the start:
|
||||
```bash
|
||||
wp mls recovery auto --quiet && wp mls sync incremental
|
||||
```
|
||||
|
||||
### Public API Functions
|
||||
|
||||
Available for themes/plugins:
|
||||
|
||||
@@ -317,6 +317,17 @@ class MLS_Sync_Engine {
|
||||
|
||||
// Check for next page
|
||||
if (isset($response['@odata.nextLink'])) {
|
||||
// Save progress for resume capability
|
||||
if (!$dry_run) {
|
||||
$this->update_sync_state(array(
|
||||
'last_next_link' => $response['@odata.nextLink'],
|
||||
'records_processed' => $this->stats['processed'],
|
||||
'records_created' => $this->stats['created'],
|
||||
'records_updated' => $this->stats['updated'],
|
||||
'records_deleted' => $this->stats['deleted'],
|
||||
));
|
||||
}
|
||||
|
||||
$start_time = microtime(true);
|
||||
$this->emit_progress('api_request', array(
|
||||
'method' => 'GET',
|
||||
@@ -809,4 +820,87 @@ class MLS_Sync_Engine {
|
||||
'stats' => $this->db->get_stats(),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get resumable (failed or interrupted) syncs
|
||||
*
|
||||
* @return array List of resumable sync states
|
||||
*/
|
||||
public function get_resumable_syncs() {
|
||||
global $wpdb;
|
||||
|
||||
// Find failed syncs that have a next_link (can be resumed)
|
||||
// Also find "running" syncs older than 1 hour (likely interrupted)
|
||||
$one_hour_ago = date('Y-m-d H:i:s', strtotime('-1 hour'));
|
||||
|
||||
return $wpdb->get_results($wpdb->prepare(
|
||||
"SELECT * FROM {$this->db->sync_state_table()}
|
||||
WHERE (status = 'failed' AND last_next_link IS NOT NULL)
|
||||
OR (status = 'running' AND updated_at < %s)
|
||||
ORDER BY updated_at DESC",
|
||||
$one_hour_ago
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the most recent resumable sync
|
||||
*
|
||||
* @return object|null Sync state or null
|
||||
*/
|
||||
public function get_latest_resumable() {
|
||||
$resumable = $this->get_resumable_syncs();
|
||||
return !empty($resumable) ? $resumable[0] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark stale running syncs as failed
|
||||
* Call this on startup to clean up interrupted syncs
|
||||
*
|
||||
* @param int $hours_threshold Hours after which running sync is considered stale
|
||||
* @return int Number of syncs marked as failed
|
||||
*/
|
||||
public function cleanup_stale_syncs($hours_threshold = 1) {
|
||||
global $wpdb;
|
||||
|
||||
$threshold_time = date('Y-m-d H:i:s', strtotime("-{$hours_threshold} hour"));
|
||||
|
||||
$updated = $wpdb->query($wpdb->prepare(
|
||||
"UPDATE {$this->db->sync_state_table()}
|
||||
SET status = 'failed', last_error = 'Sync interrupted (stale)'
|
||||
WHERE status = 'running' AND updated_at < %s",
|
||||
$threshold_time
|
||||
));
|
||||
|
||||
if ($updated > 0) {
|
||||
$this->logger->info('Cleaned up stale syncs', array('count' => $updated));
|
||||
}
|
||||
|
||||
return $updated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-resume the most recent failed/interrupted sync
|
||||
*
|
||||
* @param callable|null $progress_callback Progress callback
|
||||
* @return array|null Sync results or null if nothing to resume
|
||||
*/
|
||||
public function auto_resume($progress_callback = null) {
|
||||
// First clean up any stale syncs
|
||||
$this->cleanup_stale_syncs();
|
||||
|
||||
// Find the most recent resumable sync
|
||||
$resumable = $this->get_latest_resumable();
|
||||
|
||||
if (!$resumable) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$this->logger->info('Auto-resuming sync', array(
|
||||
'sync_id' => $resumable->id,
|
||||
'type' => $resumable->sync_type,
|
||||
'processed' => $resumable->records_processed,
|
||||
));
|
||||
|
||||
return $this->resume_sync($resumable->id, $progress_callback);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user