Add queue-based media download system with rate limiting
- Add download_status, retry_after, queued_at columns to mls_media table - Add mls_media_log table for download attempt tracking - Rewrite media handler to queue downloads instead of immediate download - Add 700ms delay between downloads (25% buffer over 2/sec limit) - Add 3-hour backoff for rate-limited (429) responses - Add max 5 attempts before marking as permanently failed - Add wp mls media command: status, process, reset, logs - Deprecate wp mls sync media in favor of wp mls media process - Update documentation with queue system details and cron examples Media downloads are now separate from property sync: 1. wp mls sync full/incremental - syncs properties, queues media 2. wp mls media process - downloads queued media with rate limiting Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Hanson.xyz Dev
@@ -37,6 +37,7 @@ class MLS_CLI {
|
||||
WP_CLI::add_command('mls stats', array($instance, 'stats'));
|
||||
WP_CLI::add_command('mls cache', array($instance, 'cache'));
|
||||
WP_CLI::add_command('mls recovery', array($instance, 'recovery'));
|
||||
WP_CLI::add_command('mls media', array($instance, 'media'));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -337,40 +338,14 @@ class MLS_CLI {
|
||||
break;
|
||||
|
||||
case 'media':
|
||||
WP_CLI::line('Downloading pending media...');
|
||||
if (!$quiet) {
|
||||
WP_CLI::line('Legend: P=downloaded p=skipped E=error');
|
||||
echo "\n";
|
||||
}
|
||||
|
||||
$media_handler = $this->plugin->get_media_handler();
|
||||
$result = $media_handler->download_pending($limit ?: 100, $progress_callback);
|
||||
|
||||
if (!$quiet) {
|
||||
echo "\n";
|
||||
}
|
||||
WP_CLI::line(sprintf(
|
||||
'Media download complete: %d success, %d failed out of %d total',
|
||||
$result['success'],
|
||||
$result['failed'],
|
||||
$result['total']
|
||||
// Redirect to the new media command
|
||||
WP_CLI::line('Note: "wp mls sync media" is deprecated. Use "wp mls media process" instead.');
|
||||
WP_CLI::line('');
|
||||
$this->media(array('process'), array(
|
||||
'limit' => $limit ?: 100,
|
||||
'verbose' => $verbose,
|
||||
'quiet' => $quiet,
|
||||
));
|
||||
|
||||
if ($result['failed'] === 0 && $result['total'] > 0) {
|
||||
WP_CLI::success('All media downloaded successfully!');
|
||||
} elseif ($result['total'] === 0) {
|
||||
WP_CLI::success('No pending media to download.');
|
||||
} else {
|
||||
WP_CLI::warning('Some media failed to download.');
|
||||
$missing_count = $media_handler->get_missing_count();
|
||||
if ($missing_count > 0) {
|
||||
WP_CLI::line(sprintf(
|
||||
'Missing media log: %s (%d entries)',
|
||||
$media_handler->get_missing_log_path(),
|
||||
$missing_count
|
||||
));
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case 'resume':
|
||||
@@ -823,6 +798,267 @@ class MLS_CLI {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Manage media download queue.
|
||||
*
|
||||
* ## OPTIONS
|
||||
*
|
||||
* <action>
|
||||
* : Action: queue, process, status, reset, logs
|
||||
*
|
||||
* [--limit=<n>]
|
||||
* : Limit number of items to process
|
||||
*
|
||||
* [--verbose]
|
||||
* : Show detailed output
|
||||
*
|
||||
* [--quiet]
|
||||
* : Suppress progress output
|
||||
*
|
||||
* [--days=<n>]
|
||||
* : Days of logs to keep (for logs --clear)
|
||||
*
|
||||
* [--clear]
|
||||
* : Clear logs older than --days
|
||||
*
|
||||
* ## EXAMPLES
|
||||
*
|
||||
* wp mls media status # Show queue statistics
|
||||
* wp mls media process # Process pending downloads (rate limited)
|
||||
* wp mls media process --limit=50 # Process up to 50 items
|
||||
* wp mls media reset # Reset failed downloads for retry
|
||||
* wp mls media logs # Show recent download logs
|
||||
* wp mls media logs --clear --days=7 # Clear logs older than 7 days
|
||||
*
|
||||
* @subcommand media
|
||||
*/
|
||||
public function media($args, $assoc_args) {
|
||||
$action = isset($args[0]) ? $args[0] : 'status';
|
||||
$limit = isset($assoc_args['limit']) ? (int) $assoc_args['limit'] : 100;
|
||||
$verbose = isset($assoc_args['verbose']);
|
||||
$quiet = isset($assoc_args['quiet']);
|
||||
|
||||
$media_handler = $this->plugin->get_media_handler();
|
||||
|
||||
switch ($action) {
|
||||
case 'status':
|
||||
case 'queue':
|
||||
$stats = $media_handler->get_queue_stats();
|
||||
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line('=== Media Download Queue ===');
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line(sprintf('Pending total: %d', $stats['pending']));
|
||||
WP_CLI::line(sprintf('Ready now: %d', $stats['ready']));
|
||||
WP_CLI::line(sprintf('In backoff: %d (retry scheduled)', $stats['in_backoff']));
|
||||
WP_CLI::line(sprintf('Failed: %d (max attempts reached)', $stats['failed']));
|
||||
WP_CLI::line(sprintf('Completed: %d', $stats['completed']));
|
||||
WP_CLI::line('');
|
||||
|
||||
if ($stats['ready'] > 0) {
|
||||
WP_CLI::line(sprintf(
|
||||
'Run "wp mls media process --limit=%d" to download pending media.',
|
||||
min($stats['ready'], 100)
|
||||
));
|
||||
WP_CLI::line(sprintf(
|
||||
'Estimated time: %d minutes (at 700ms per image)',
|
||||
ceil($stats['ready'] * 0.7 / 60)
|
||||
));
|
||||
}
|
||||
|
||||
if ($stats['failed'] > 0) {
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line('Run "wp mls media reset" to retry failed downloads.');
|
||||
}
|
||||
WP_CLI::line('');
|
||||
break;
|
||||
|
||||
case 'process':
|
||||
$stats = $media_handler->get_queue_stats();
|
||||
|
||||
if ($stats['ready'] === 0) {
|
||||
WP_CLI::success('No media ready to download.');
|
||||
break;
|
||||
}
|
||||
|
||||
$process_count = min($limit, $stats['ready']);
|
||||
|
||||
WP_CLI::line(sprintf(
|
||||
'Processing %d media items (rate limited: 1 per 700ms)...',
|
||||
$process_count
|
||||
));
|
||||
WP_CLI::line(sprintf(
|
||||
'Estimated time: %d minutes',
|
||||
ceil($process_count * 0.7 / 60)
|
||||
));
|
||||
|
||||
if (!$quiet) {
|
||||
WP_CLI::line('Legend: P=downloaded B=backoff (retry later) E=error');
|
||||
echo "\n";
|
||||
}
|
||||
|
||||
// Progress callback
|
||||
$progress_callback = null;
|
||||
if (!$quiet) {
|
||||
$progress_callback = function($event, $data = array()) use ($verbose) {
|
||||
if ($verbose) {
|
||||
$this->output_verbose_media_event($event, $data);
|
||||
} else {
|
||||
switch ($event) {
|
||||
case 'media_downloaded':
|
||||
echo 'P';
|
||||
break;
|
||||
case 'media_backoff':
|
||||
echo 'B';
|
||||
break;
|
||||
case 'media_error':
|
||||
echo 'E';
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
$result = $media_handler->process_queue($process_count, $progress_callback);
|
||||
|
||||
if (!$quiet) {
|
||||
echo "\n\n";
|
||||
}
|
||||
|
||||
WP_CLI::line(sprintf(
|
||||
'Results: %d success, %d backoff, %d failed out of %d processed',
|
||||
$result['success'],
|
||||
$result['skipped'],
|
||||
$result['failed'],
|
||||
$result['processed']
|
||||
));
|
||||
|
||||
// Show updated stats
|
||||
$new_stats = $media_handler->get_queue_stats();
|
||||
WP_CLI::line(sprintf('Queue remaining: %d ready, %d in backoff', $new_stats['ready'], $new_stats['in_backoff']));
|
||||
|
||||
if ($result['failed'] > 0 || $result['skipped'] > 0) {
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line('Items in backoff will be retried after 3 hours.');
|
||||
WP_CLI::line('Run "wp mls media logs" to see download history.');
|
||||
}
|
||||
|
||||
if ($result['success'] > 0) {
|
||||
WP_CLI::success('Media processing complete.');
|
||||
}
|
||||
break;
|
||||
|
||||
case 'reset':
|
||||
WP_CLI::line('Resetting failed downloads for retry...');
|
||||
|
||||
$reset_count = $media_handler->reset_failed_downloads();
|
||||
|
||||
if ($reset_count > 0) {
|
||||
WP_CLI::success(sprintf('Reset %d failed downloads. They will be retried on next process.', $reset_count));
|
||||
} else {
|
||||
WP_CLI::success('No failed downloads to reset.');
|
||||
}
|
||||
break;
|
||||
|
||||
case 'logs':
|
||||
if (isset($assoc_args['clear'])) {
|
||||
$days = isset($assoc_args['days']) ? (int) $assoc_args['days'] : 7;
|
||||
$deleted = $media_handler->clear_old_logs($days);
|
||||
WP_CLI::success(sprintf('Deleted %d log entries older than %d days.', $deleted, $days));
|
||||
break;
|
||||
}
|
||||
|
||||
$logs = $media_handler->get_download_logs($limit);
|
||||
|
||||
if (empty($logs)) {
|
||||
WP_CLI::success('No download logs found.');
|
||||
break;
|
||||
}
|
||||
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line('=== Recent Download Logs ===');
|
||||
WP_CLI::line('');
|
||||
|
||||
foreach ($logs as $log) {
|
||||
$status_indicator = '';
|
||||
switch ($log->action) {
|
||||
case 'success':
|
||||
$status_indicator = '[OK]';
|
||||
break;
|
||||
case 'rate_limited':
|
||||
$status_indicator = '[429]';
|
||||
break;
|
||||
case 'permanent_error':
|
||||
$status_indicator = '[ERR]';
|
||||
break;
|
||||
case 'error':
|
||||
$status_indicator = '[FAIL]';
|
||||
break;
|
||||
default:
|
||||
$status_indicator = "[{$log->action}]";
|
||||
}
|
||||
|
||||
$line = sprintf(
|
||||
'%s %s %s %s %dms',
|
||||
$log->created_at,
|
||||
$status_indicator,
|
||||
$log->listing_key,
|
||||
$log->media_key,
|
||||
$log->response_time_ms
|
||||
);
|
||||
|
||||
if ($log->status_code) {
|
||||
$line .= " HTTP:{$log->status_code}";
|
||||
}
|
||||
|
||||
if ($log->error_message) {
|
||||
$line .= " - {$log->error_message}";
|
||||
}
|
||||
|
||||
WP_CLI::line($line);
|
||||
}
|
||||
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line(sprintf('Showing %d most recent entries. Use --limit=N to see more.', count($logs)));
|
||||
WP_CLI::line('');
|
||||
break;
|
||||
|
||||
default:
|
||||
WP_CLI::error("Unknown action: {$action}. Use 'status', 'process', 'reset', or 'logs'.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Output verbose media event information
|
||||
*
|
||||
* @param string $event Event name
|
||||
* @param array $data Event data
|
||||
*/
|
||||
private function output_verbose_media_event($event, $data) {
|
||||
$timestamp = date('H:i:s');
|
||||
|
||||
switch ($event) {
|
||||
case 'media_downloaded':
|
||||
$listing = $data['listing_key'] ?? 'unknown';
|
||||
$key = $data['media_key'] ?? 'unknown';
|
||||
WP_CLI::line("[{$timestamp}] DOWNLOADED: {$listing} / {$key}");
|
||||
break;
|
||||
|
||||
case 'media_backoff':
|
||||
$listing = $data['listing_key'] ?? 'unknown';
|
||||
$key = $data['media_key'] ?? 'unknown';
|
||||
WP_CLI::warning("[{$timestamp}] BACKOFF: {$listing} / {$key} - will retry in 3 hours");
|
||||
break;
|
||||
|
||||
case 'media_error':
|
||||
$listing = $data['listing_key'] ?? 'unknown';
|
||||
$key = $data['media_key'] ?? 'unknown';
|
||||
$error = $data['error'] ?? 'Unknown error';
|
||||
WP_CLI::error("[{$timestamp}] ERROR: {$listing} / {$key} - {$error}", false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively delete a directory
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user