brianhansonxyz/homeproz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add verbose mode, progress indicators, and missing media log

Browse Source 
- Add --verbose flag to sync commands for detailed API request/response output
- Add progress indicators (.=#xPpE|) for compact sync output
- Implement exponential backoff (1s, 2s, 4s, 8s, 16s) for media downloads
- Log failed media downloads to wp-content/uploads/mls-missing-media.log
- Add 'wp mls cache missing' command to view/clear the log
- Retry on rate limit (429) and server errors (5xx)
- Update documentation with new features

Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Hanson.xyz Dev
2025-12-14 22:20:41 -06:00
parent 6556479417
commit 5e4ebfb99e
5 changed files with 626 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
wp-content/plugins/mls-by-hansonxyz/includes/class-mls-media-handler.php
+168 -18
View File
@@ -72,8 +72,9 @@ class MLS_Media_Handler {
* @param string $listing_key Listing key
* @param array $media_array Media array from API
* @param bool $force Force re-download all media
* @param callable|null $progress_callback Callback for progress updates
*/
public function sync_property_media($listing_key, $media_array, $force = false) {
public function sync_property_media($listing_key, $media_array, $force = false, $progress_callback = null) {
global $wpdb;
if (empty($media_array)) {
@@ -122,15 +123,36 @@ class MLS_Media_Handler {
// Check if we need to re-download
if ($force || $this->needs_download($existing, $media)) {
$this->download_media($existing->id);
$result = $this->download_media($existing->id);
if ($progress_callback) {
if ($result) {
call_user_func($progress_callback, 'media_downloaded', array('media_key' => $media_key));
} else {
$error = $this->get_last_download_error($existing->id);
call_user_func($progress_callback, 'media_error', array('media_key' => $media_key, 'error' => $error));
}
}
} else {
if ($progress_callback) {
call_user_func($progress_callback, 'media_skipped', array('media_key' => $media_key));
}
}
} else {
// Insert new record
$data['created_at'] = current_time('mysql');
$wpdb->insert($this->db->media_table(), $data);
$new_id = $wpdb->insert_id;
// Queue download
$this->download_media($wpdb->insert_id);
$result = $this->download_media($new_id);
if ($progress_callback) {
if ($result) {
call_user_func($progress_callback, 'media_downloaded', array('media_key' => $media_key));
} else {
$error = $this->get_last_download_error($new_id);
call_user_func($progress_callback, 'media_error', array('media_key' => $media_key, 'error' => $error));
}
}
}
}
@@ -160,6 +182,74 @@ class MLS_Media_Handler {
}
}
/**
* Get the last download error for a media record
*
* @param int $media_id Media ID
* @return string|null Error message
*/
private function get_last_download_error($media_id) {
global $wpdb;
return $wpdb->get_var($wpdb->prepare(
"SELECT download_error FROM {$this->db->media_table()} WHERE id = %d",
$media_id
));
}
/**
* Get the path to the missing media log file
*
* @return string File path
*/
public function get_missing_log_path() {
$upload_dir = wp_upload_dir();
return $upload_dir['basedir'] . '/mls-missing-media.log';
}
/**
* Log a failed media download to the missing media log file
*
* @param object $media Media record
* @param string $error Error message
*/
private function log_missing_media($media, $error) {
$log_file = $this->get_missing_log_path();
$timestamp = date('Y-m-d H:i:s');
$line = sprintf(
"[%s] %s | %s | %s | %s\n",
$timestamp,
$media->listing_key,
$media->media_key,
$error,
$media->media_url
);
file_put_contents($log_file, $line, FILE_APPEND | LOCK_EX);
}
/**
* Clear the missing media log file
*/
public function clear_missing_log() {
$log_file = $this->get_missing_log_path();
if (file_exists($log_file)) {
unlink($log_file);
}
}
/**
* Get missing media count from log file
*
* @return int Number of missing media entries
*/
public function get_missing_count() {
$log_file = $this->get_missing_log_path();
if (!file_exists($log_file)) {
return 0;
}
$content = file_get_contents($log_file);
return substr_count($content, "\n");
}
/**
* Check if media needs to be downloaded
*
@@ -212,34 +302,87 @@ class MLS_Media_Handler {
array('id' => $media_id)
);
// Download file
$response = wp_remote_get($media->media_url, array(
'timeout' => 60,
'stream' => false,
));
// Download with exponential backoff for rate limits
$max_retries = 5;
$response = null;
$status_code = 0;
$base_delay = 1; // Start with 1 second
if (is_wp_error($response)) {
$this->logger->warning('Media download failed', array(
'media_id' => $media_id,
'error' => $response->get_error_message(),
for ($retry = 0; $retry < $max_retries; $retry++) {
// Exponential backoff: 1s, 2s, 4s, 8s, 16s
if ($retry > 0) {
$delay = $base_delay * pow(2, $retry - 1);
$this->logger->debug('Media download retry', array(
'media_id' => $media_id,
'retry' => $retry,
'delay' => $delay,
));
sleep($delay);
}
$response = wp_remote_get($media->media_url, array(
'timeout' => 60,
'stream' => false,
));
if (is_wp_error($response)) {
$error_msg = $response->get_error_message();
$this->logger->warning('Media download failed', array(
'media_id' => $media_id,
'error' => $error_msg,
'retry' => $retry,
));
if ($retry === $max_retries - 1) {
$wpdb->update(
$this->db->media_table(),
array('download_error' => $error_msg),
array('id' => $media_id)
);
$this->log_missing_media($media, $error_msg);
return false;
}
continue;
}
$status_code = wp_remote_retrieve_response_code($response);
// Success
if ($status_code === 200) {
break;
}
// Retryable errors: 429 (rate limit), 500, 502, 503, 504 (server errors)
$retryable = in_array($status_code, array(429, 500, 502, 503, 504));
if ($retryable && $retry < $max_retries - 1) {
$this->logger->debug('Media download retryable error', array(
'media_id' => $media_id,
'status_code' => $status_code,
'retry' => $retry,
));
continue;
}
// Non-retryable or exhausted retries - record and fail
$error_msg = "HTTP {$status_code}";
$wpdb->update(
$this->db->media_table(),
array('download_error' => $response->get_error_message()),
array('download_error' => $error_msg),
array('id' => $media_id)
);
$this->log_missing_media($media, $error_msg);
return false;
}
$status_code = wp_remote_retrieve_response_code($response);
if ($status_code !== 200) {
$error_msg = "HTTP {$status_code}";
$wpdb->update(
$this->db->media_table(),
array('download_error' => "HTTP {$status_code}"),
array('download_error' => $error_msg),
array('id' => $media_id)
);
$this->log_missing_media($media, $error_msg);
return false;
}
@@ -418,13 +561,14 @@ class MLS_Media_Handler {
* Download pending media (for batch processing)
*
* @param int $limit Max media to download
* @param callable|null $progress_callback Callback for progress updates
* @return array Stats
*/
public function download_pending($limit = 100) {
public function download_pending($limit = 100, $progress_callback = null) {
global $wpdb;
$pending = $wpdb->get_results($wpdb->prepare(
"SELECT id FROM {$this->db->media_table()}
"SELECT id, media_key FROM {$this->db->media_table()}
WHERE local_path IS NULL AND media_url IS NOT NULL
AND download_attempts < 3
LIMIT %d",
@@ -440,8 +584,14 @@ class MLS_Media_Handler {
foreach ($pending as $media) {
if ($this->download_media($media->id)) {
$stats['success']++;
if ($progress_callback) {
call_user_func($progress_callback, 'media_downloaded', array('media_key' => $media->media_key));
}
} else {
$stats['failed']++;
if ($progress_callback) {
call_user_func($progress_callback, 'media_error', array('media_key' => $media->media_key));
}
}
}
wp-content/plugins/mls-by-hansonxyz/includes/class-mls-sync-engine.php
+131 -13
View File
@@ -88,6 +88,9 @@ class MLS_Sync_Engine {
public function run_full_sync($dry_run = false, $limit = null, $progress_callback = null) {
$this->logger->info('Starting full sync', array('dry_run' => $dry_run, 'limit' => $limit));
// Store progress callback for use in process_property
$this->progress_callback = $progress_callback;
// Create sync state record
if (!$dry_run) {
$this->sync_state_id = $this->create_sync_state(self::TYPE_FULL);
@@ -104,12 +107,35 @@ class MLS_Sync_Engine {
try {
// Get first page of properties with media
$start_time = microtime(true);
$this->emit_progress('api_request', array(
'method' => 'GET',
'url' => 'Property',
'params' => array('type' => 'full_sync', 'limit' => $limit),
));
$response = $this->api_client->get_properties_for_sync(null, 'Media', $limit ? min($limit, 1000) : null);
$elapsed = round((microtime(true) - $start_time) * 1000);
if (is_wp_error($response)) {
$this->emit_progress('api_response', array(
'success' => false,
'status_code' => 0,
'error' => $response->get_error_message(),
'record_count' => 0,
'response_time' => $elapsed,
));
throw new Exception($response->get_error_message());
}
$this->emit_progress('api_response', array(
'success' => true,
'status_code' => 200,
'record_count' => isset($response['value']) ? count($response['value']) : 0,
'response_time' => $elapsed,
'has_more' => isset($response['@odata.nextLink']),
));
// Process pages
$continue = true;
while ($continue && isset($response['value'])) {
@@ -120,12 +146,11 @@ class MLS_Sync_Engine {
}
$this->process_property($property, $dry_run);
if ($progress_callback) {
call_user_func($progress_callback, $this->stats);
}
}
// Emit page complete event
$this->emit_progress('page_complete', array('processed' => $this->stats['processed']));
// Check for next page
if ($continue && isset($response['@odata.nextLink'])) {
// Save progress
@@ -138,11 +163,34 @@ class MLS_Sync_Engine {
));
}
$start_time = microtime(true);
$this->emit_progress('api_request', array(
'method' => 'GET',
'url' => 'Property (next page)',
'params' => array('page' => 'next'),
));
$response = $this->api_client->get_next_page($response['@odata.nextLink']);
$elapsed = round((microtime(true) - $start_time) * 1000);
if (is_wp_error($response)) {
$this->emit_progress('api_response', array(
'success' => false,
'status_code' => 0,
'error' => $response->get_error_message(),
'record_count' => 0,
'response_time' => $elapsed,
));
throw new Exception($response->get_error_message());
}
$this->emit_progress('api_response', array(
'success' => true,
'status_code' => 200,
'record_count' => isset($response['value']) ? count($response['value']) : 0,
'response_time' => $elapsed,
'has_more' => isset($response['@odata.nextLink']),
));
} else {
$continue = false;
}
@@ -211,6 +259,9 @@ class MLS_Sync_Engine {
'dry_run' => $dry_run,
));
// Store progress callback for use in process_property
$this->progress_callback = $progress_callback;
if (!$dry_run) {
$this->sync_state_id = $this->create_sync_state(self::TYPE_INCREMENTAL);
$this->logger->set_sync_state($this->sync_state_id);
@@ -226,29 +277,74 @@ class MLS_Sync_Engine {
try {
// Get modified properties (including those marked for deletion)
$start_time = microtime(true);
$this->emit_progress('api_request', array(
'method' => 'GET',
'url' => 'Property',
'params' => array('type' => 'incremental', 'since' => $last_timestamp),
));
$response = $this->api_client->get_properties_since($last_timestamp, 'Media');
$elapsed = round((microtime(true) - $start_time) * 1000);
if (is_wp_error($response)) {
$this->emit_progress('api_response', array(
'success' => false,
'status_code' => 0,
'error' => $response->get_error_message(),
'record_count' => 0,
'response_time' => $elapsed,
));
throw new Exception($response->get_error_message());
}
$this->emit_progress('api_response', array(
'success' => true,
'status_code' => 200,
'record_count' => isset($response['value']) ? count($response['value']) : 0,
'response_time' => $elapsed,
'has_more' => isset($response['@odata.nextLink']),
));
// Process pages
while (isset($response['value'])) {
foreach ($response['value'] as $property) {
$this->process_property($property, $dry_run);
if ($progress_callback) {
call_user_func($progress_callback, $this->stats);
}
}
// Emit page complete event
$this->emit_progress('page_complete', array('processed' => $this->stats['processed']));
// Check for next page
if (isset($response['@odata.nextLink'])) {
$start_time = microtime(true);
$this->emit_progress('api_request', array(
'method' => 'GET',
'url' => 'Property (next page)',
'params' => array('page' => 'next'),
));
$response = $this->api_client->get_next_page($response['@odata.nextLink']);
$elapsed = round((microtime(true) - $start_time) * 1000);
if (is_wp_error($response)) {
$this->emit_progress('api_response', array(
'success' => false,
'status_code' => 0,
'error' => $response->get_error_message(),
'record_count' => 0,
'response_time' => $elapsed,
));
throw new Exception($response->get_error_message());
}
$this->emit_progress('api_response', array(
'success' => true,
'status_code' => 200,
'record_count' => isset($response['value']) ? count($response['value']) : 0,
'response_time' => $elapsed,
'has_more' => isset($response['@odata.nextLink']),
));
} else {
break;
}
@@ -324,6 +420,7 @@ class MLS_Sync_Engine {
}
$this->sync_state_id = $sync_state_id;
$this->progress_callback = $progress_callback;
$this->logger->set_sync_state($sync_state_id);
$this->logger->info('Resuming sync', array('sync_state_id' => $sync_state_id));
@@ -359,12 +456,11 @@ class MLS_Sync_Engine {
while (isset($response['value'])) {
foreach ($response['value'] as $property) {
$this->process_property($property, false);
if ($progress_callback) {
call_user_func($progress_callback, $this->stats);
}
}
// Emit page complete event
$this->emit_progress('page_complete', array('processed' => $this->stats['processed']));
if (isset($response['@odata.nextLink'])) {
$this->update_sync_state(array(
'last_next_link' => $response['@odata.nextLink'],
@@ -413,6 +509,11 @@ class MLS_Sync_Engine {
);
}
/**
* Progress callback reference
*/
private $progress_callback = null;
/**
* Process a single property record
*
@@ -439,6 +540,7 @@ class MLS_Sync_Engine {
$this->delete_property($listing_key);
}
$this->stats['deleted']++;
$this->emit_progress('property_deleted', array('listing_key' => $listing_key));
return;
}
@@ -454,8 +556,10 @@ class MLS_Sync_Engine {
if ($dry_run) {
if ($existing) {
$this->stats['updated']++;
$this->emit_progress('property_skipped', array('listing_key' => $listing_key));
} else {
$this->stats['created']++;
$this->emit_progress('property_skipped', array('listing_key' => $listing_key));
}
return;
}
@@ -468,17 +572,31 @@ class MLS_Sync_Engine {
array('listing_key' => $listing_key)
);
$this->stats['updated']++;
$this->emit_progress('property_updated', array('listing_key' => $listing_key));
} else {
// Insert new
$data['listing_key'] = $listing_key;
$data['created_at'] = current_time('mysql');
$wpdb->insert($this->db->properties_table(), $data);
$this->stats['created']++;
$this->emit_progress('property_created', array('listing_key' => $listing_key));
}
// Process media if present
if (isset($property['Media']) && is_array($property['Media'])) {
$this->media_handler->sync_property_media($listing_key, $property['Media']);
$this->media_handler->sync_property_media($listing_key, $property['Media'], false, $this->progress_callback);
}
}
/**
* Emit progress event
*
* @param string $event Event name
* @param array $data Event data
*/
private function emit_progress($event, $data = array()) {
if ($this->progress_callback) {
call_user_func($this->progress_callback, $event, $data);
}
}