Add sync recovery commands for interrupted syncs
- Add wp mls recovery list to show resumable syncs - Add wp mls recovery auto to auto-resume most recent failed sync - Add wp mls recovery cleanup to mark stale syncs (>1hr) as failed - Track last_next_link during incremental sync pagination - Add get_resumable_syncs(), cleanup_stale_syncs(), auto_resume() methods 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -36,6 +36,7 @@ class MLS_CLI {
|
|||||||
WP_CLI::add_command('mls sync', array($instance, 'sync'));
|
WP_CLI::add_command('mls sync', array($instance, 'sync'));
|
||||||
WP_CLI::add_command('mls stats', array($instance, 'stats'));
|
WP_CLI::add_command('mls stats', array($instance, 'stats'));
|
||||||
WP_CLI::add_command('mls cache', array($instance, 'cache'));
|
WP_CLI::add_command('mls cache', array($instance, 'cache'));
|
||||||
|
WP_CLI::add_command('mls recovery', array($instance, 'recovery'));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -684,6 +685,144 @@ class MLS_CLI {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manage sync recovery and resumption.
|
||||||
|
*
|
||||||
|
* ## OPTIONS
|
||||||
|
*
|
||||||
|
* <action>
|
||||||
|
* : Action: list, auto, cleanup
|
||||||
|
*
|
||||||
|
* [--verbose]
|
||||||
|
* : Show detailed output during resume
|
||||||
|
*
|
||||||
|
* [--quiet]
|
||||||
|
* : Suppress progress output
|
||||||
|
*
|
||||||
|
* ## EXAMPLES
|
||||||
|
*
|
||||||
|
* wp mls recovery list # Show resumable syncs
|
||||||
|
* wp mls recovery auto # Auto-resume most recent failed sync
|
||||||
|
* wp mls recovery auto --verbose # Auto-resume with detailed output
|
||||||
|
* wp mls recovery cleanup # Mark stale syncs as failed
|
||||||
|
*
|
||||||
|
* @subcommand recovery
|
||||||
|
*/
|
||||||
|
public function recovery($args, $assoc_args) {
|
||||||
|
$action = isset($args[0]) ? $args[0] : 'list';
|
||||||
|
$verbose = isset($assoc_args['verbose']);
|
||||||
|
$quiet = isset($assoc_args['quiet']);
|
||||||
|
|
||||||
|
$sync_engine = $this->plugin->get_sync_engine();
|
||||||
|
|
||||||
|
switch ($action) {
|
||||||
|
case 'list':
|
||||||
|
$resumable = $sync_engine->get_resumable_syncs();
|
||||||
|
|
||||||
|
if (empty($resumable)) {
|
||||||
|
WP_CLI::success('No resumable syncs found.');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
WP_CLI::line('');
|
||||||
|
WP_CLI::line('=== Resumable Syncs ===');
|
||||||
|
WP_CLI::line('');
|
||||||
|
|
||||||
|
foreach ($resumable as $sync) {
|
||||||
|
$status_color = $sync->status === 'failed' ? '%R' : '%Y';
|
||||||
|
WP_CLI::line(sprintf(
|
||||||
|
'ID: %d | Type: %s | Status: %s | Processed: %d',
|
||||||
|
$sync->id,
|
||||||
|
$sync->sync_type,
|
||||||
|
$sync->status,
|
||||||
|
$sync->records_processed
|
||||||
|
));
|
||||||
|
WP_CLI::line(sprintf(
|
||||||
|
' Started: %s | Updated: %s',
|
||||||
|
$sync->started_at,
|
||||||
|
$sync->updated_at
|
||||||
|
));
|
||||||
|
if ($sync->last_error) {
|
||||||
|
WP_CLI::warning(' Error: ' . $sync->last_error);
|
||||||
|
}
|
||||||
|
if ($sync->last_next_link) {
|
||||||
|
WP_CLI::line(' Has resume point: Yes');
|
||||||
|
}
|
||||||
|
WP_CLI::line('');
|
||||||
|
}
|
||||||
|
|
||||||
|
WP_CLI::line('To resume a specific sync: wp mls sync resume --id=<ID>');
|
||||||
|
WP_CLI::line('To auto-resume the most recent: wp mls recovery auto');
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'auto':
|
||||||
|
WP_CLI::line('Checking for resumable syncs...');
|
||||||
|
|
||||||
|
// Build progress callback
|
||||||
|
$progress_callback = null;
|
||||||
|
if (!$quiet) {
|
||||||
|
$progress_callback = function($event, $data = array()) use ($verbose) {
|
||||||
|
if ($verbose) {
|
||||||
|
$this->output_verbose_event($event, $data);
|
||||||
|
} else {
|
||||||
|
switch ($event) {
|
||||||
|
case 'property_created':
|
||||||
|
echo '.';
|
||||||
|
break;
|
||||||
|
case 'property_updated':
|
||||||
|
echo '#';
|
||||||
|
break;
|
||||||
|
case 'property_deleted':
|
||||||
|
echo 'x';
|
||||||
|
break;
|
||||||
|
case 'media_downloaded':
|
||||||
|
echo 'P';
|
||||||
|
break;
|
||||||
|
case 'media_skipped':
|
||||||
|
echo 'p';
|
||||||
|
break;
|
||||||
|
case 'media_error':
|
||||||
|
echo 'E';
|
||||||
|
break;
|
||||||
|
case 'page_complete':
|
||||||
|
echo '|';
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
$result = $sync_engine->auto_resume($progress_callback);
|
||||||
|
|
||||||
|
if ($result === null) {
|
||||||
|
WP_CLI::success('No syncs to resume.');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$quiet && !$verbose) {
|
||||||
|
echo "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->output_sync_result($result);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'cleanup':
|
||||||
|
WP_CLI::line('Cleaning up stale syncs...');
|
||||||
|
|
||||||
|
$cleaned = $sync_engine->cleanup_stale_syncs();
|
||||||
|
|
||||||
|
if ($cleaned > 0) {
|
||||||
|
WP_CLI::success("Marked {$cleaned} stale sync(s) as failed.");
|
||||||
|
} else {
|
||||||
|
WP_CLI::success('No stale syncs found.');
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
WP_CLI::error("Unknown action: {$action}. Use 'list', 'auto', or 'cleanup'.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Recursively delete a directory
|
* Recursively delete a directory
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -76,6 +76,11 @@ wp mls cache cleanup
|
|||||||
wp mls cache missing # View failed media downloads
|
wp mls cache missing # View failed media downloads
|
||||||
wp mls cache missing --limit=20 # View first 20 entries
|
wp mls cache missing --limit=20 # View first 20 entries
|
||||||
wp mls cache missing --clear # Clear the log
|
wp mls cache missing --clear # Clear the log
|
||||||
|
|
||||||
|
# Recovery commands
|
||||||
|
wp mls recovery list # Show resumable syncs
|
||||||
|
wp mls recovery auto # Auto-resume most recent failed sync
|
||||||
|
wp mls recovery cleanup # Mark stale (>1hr) syncs as failed
|
||||||
```
|
```
|
||||||
|
|
||||||
### Progress Output
|
### Progress Output
|
||||||
@@ -100,6 +105,22 @@ Format: `[timestamp] listing_key | media_key | error | url`
|
|||||||
|
|
||||||
Media downloads use exponential backoff (1s, 2s, 4s, 8s, 16s) for rate limit (429) and server errors (5xx).
|
Media downloads use exponential backoff (1s, 2s, 4s, 8s, 16s) for rate limit (429) and server errors (5xx).
|
||||||
|
|
||||||
|
### Sync Recovery
|
||||||
|
|
||||||
|
The sync engine saves progress after each page, allowing interrupted syncs to resume:
|
||||||
|
|
||||||
|
1. **Automatic state tracking**: `last_next_link` saved after each API page
|
||||||
|
2. **Stale sync detection**: Syncs running >1 hour marked as failed
|
||||||
|
3. **Resume commands**:
|
||||||
|
- `wp mls sync resume --id=<ID>` - Resume specific sync
|
||||||
|
- `wp mls recovery auto` - Auto-resume most recent failed sync
|
||||||
|
- `wp mls recovery list` - View all resumable syncs
|
||||||
|
|
||||||
|
For cron jobs, consider adding recovery at the start:
|
||||||
|
```bash
|
||||||
|
wp mls recovery auto --quiet && wp mls sync incremental
|
||||||
|
```
|
||||||
|
|
||||||
### Public API Functions
|
### Public API Functions
|
||||||
|
|
||||||
Available for themes/plugins:
|
Available for themes/plugins:
|
||||||
|
|||||||
@@ -317,6 +317,17 @@ class MLS_Sync_Engine {
|
|||||||
|
|
||||||
// Check for next page
|
// Check for next page
|
||||||
if (isset($response['@odata.nextLink'])) {
|
if (isset($response['@odata.nextLink'])) {
|
||||||
|
// Save progress for resume capability
|
||||||
|
if (!$dry_run) {
|
||||||
|
$this->update_sync_state(array(
|
||||||
|
'last_next_link' => $response['@odata.nextLink'],
|
||||||
|
'records_processed' => $this->stats['processed'],
|
||||||
|
'records_created' => $this->stats['created'],
|
||||||
|
'records_updated' => $this->stats['updated'],
|
||||||
|
'records_deleted' => $this->stats['deleted'],
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
$start_time = microtime(true);
|
$start_time = microtime(true);
|
||||||
$this->emit_progress('api_request', array(
|
$this->emit_progress('api_request', array(
|
||||||
'method' => 'GET',
|
'method' => 'GET',
|
||||||
@@ -809,4 +820,87 @@ class MLS_Sync_Engine {
|
|||||||
'stats' => $this->db->get_stats(),
|
'stats' => $this->db->get_stats(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get resumable (failed or interrupted) syncs
|
||||||
|
*
|
||||||
|
* @return array List of resumable sync states
|
||||||
|
*/
|
||||||
|
public function get_resumable_syncs() {
|
||||||
|
global $wpdb;
|
||||||
|
|
||||||
|
// Find failed syncs that have a next_link (can be resumed)
|
||||||
|
// Also find "running" syncs older than 1 hour (likely interrupted)
|
||||||
|
$one_hour_ago = date('Y-m-d H:i:s', strtotime('-1 hour'));
|
||||||
|
|
||||||
|
return $wpdb->get_results($wpdb->prepare(
|
||||||
|
"SELECT * FROM {$this->db->sync_state_table()}
|
||||||
|
WHERE (status = 'failed' AND last_next_link IS NOT NULL)
|
||||||
|
OR (status = 'running' AND updated_at < %s)
|
||||||
|
ORDER BY updated_at DESC",
|
||||||
|
$one_hour_ago
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the most recent resumable sync
|
||||||
|
*
|
||||||
|
* @return object|null Sync state or null
|
||||||
|
*/
|
||||||
|
public function get_latest_resumable() {
|
||||||
|
$resumable = $this->get_resumable_syncs();
|
||||||
|
return !empty($resumable) ? $resumable[0] : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark stale running syncs as failed
|
||||||
|
* Call this on startup to clean up interrupted syncs
|
||||||
|
*
|
||||||
|
* @param int $hours_threshold Hours after which running sync is considered stale
|
||||||
|
* @return int Number of syncs marked as failed
|
||||||
|
*/
|
||||||
|
public function cleanup_stale_syncs($hours_threshold = 1) {
|
||||||
|
global $wpdb;
|
||||||
|
|
||||||
|
$threshold_time = date('Y-m-d H:i:s', strtotime("-{$hours_threshold} hour"));
|
||||||
|
|
||||||
|
$updated = $wpdb->query($wpdb->prepare(
|
||||||
|
"UPDATE {$this->db->sync_state_table()}
|
||||||
|
SET status = 'failed', last_error = 'Sync interrupted (stale)'
|
||||||
|
WHERE status = 'running' AND updated_at < %s",
|
||||||
|
$threshold_time
|
||||||
|
));
|
||||||
|
|
||||||
|
if ($updated > 0) {
|
||||||
|
$this->logger->info('Cleaned up stale syncs', array('count' => $updated));
|
||||||
|
}
|
||||||
|
|
||||||
|
return $updated;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Auto-resume the most recent failed/interrupted sync
|
||||||
|
*
|
||||||
|
* @param callable|null $progress_callback Progress callback
|
||||||
|
* @return array|null Sync results or null if nothing to resume
|
||||||
|
*/
|
||||||
|
public function auto_resume($progress_callback = null) {
|
||||||
|
// First clean up any stale syncs
|
||||||
|
$this->cleanup_stale_syncs();
|
||||||
|
|
||||||
|
// Find the most recent resumable sync
|
||||||
|
$resumable = $this->get_latest_resumable();
|
||||||
|
|
||||||
|
if (!$resumable) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->logger->info('Auto-resuming sync', array(
|
||||||
|
'sync_id' => $resumable->id,
|
||||||
|
'type' => $resumable->sync_type,
|
||||||
|
'processed' => $resumable->records_processed,
|
||||||
|
));
|
||||||
|
|
||||||
|
return $this->resume_sync($resumable->id, $progress_callback);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user