Add verbose mode, progress indicators, and missing media log
- Add --verbose flag to sync commands for detailed API request/response output - Add progress indicators (.=#xPpE|) for compact sync output - Implement exponential backoff (1s, 2s, 4s, 8s, 16s) for media downloads - Log failed media downloads to wp-content/uploads/mls-missing-media.log - Add 'wp mls cache missing' command to view/clear the log - Retry on rate limit (429) and server errors (5xx) - Update documentation with new features Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Hanson.xyz Dev
@@ -88,6 +88,9 @@ class MLS_Sync_Engine {
|
||||
public function run_full_sync($dry_run = false, $limit = null, $progress_callback = null) {
|
||||
$this->logger->info('Starting full sync', array('dry_run' => $dry_run, 'limit' => $limit));
|
||||
|
||||
// Store progress callback for use in process_property
|
||||
$this->progress_callback = $progress_callback;
|
||||
|
||||
// Create sync state record
|
||||
if (!$dry_run) {
|
||||
$this->sync_state_id = $this->create_sync_state(self::TYPE_FULL);
|
||||
@@ -104,12 +107,35 @@ class MLS_Sync_Engine {
|
||||
|
||||
try {
|
||||
// Get first page of properties with media
|
||||
$start_time = microtime(true);
|
||||
$this->emit_progress('api_request', array(
|
||||
'method' => 'GET',
|
||||
'url' => 'Property',
|
||||
'params' => array('type' => 'full_sync', 'limit' => $limit),
|
||||
));
|
||||
|
||||
$response = $this->api_client->get_properties_for_sync(null, 'Media', $limit ? min($limit, 1000) : null);
|
||||
$elapsed = round((microtime(true) - $start_time) * 1000);
|
||||
|
||||
if (is_wp_error($response)) {
|
||||
$this->emit_progress('api_response', array(
|
||||
'success' => false,
|
||||
'status_code' => 0,
|
||||
'error' => $response->get_error_message(),
|
||||
'record_count' => 0,
|
||||
'response_time' => $elapsed,
|
||||
));
|
||||
throw new Exception($response->get_error_message());
|
||||
}
|
||||
|
||||
$this->emit_progress('api_response', array(
|
||||
'success' => true,
|
||||
'status_code' => 200,
|
||||
'record_count' => isset($response['value']) ? count($response['value']) : 0,
|
||||
'response_time' => $elapsed,
|
||||
'has_more' => isset($response['@odata.nextLink']),
|
||||
));
|
||||
|
||||
// Process pages
|
||||
$continue = true;
|
||||
while ($continue && isset($response['value'])) {
|
||||
@@ -120,12 +146,11 @@ class MLS_Sync_Engine {
|
||||
}
|
||||
|
||||
$this->process_property($property, $dry_run);
|
||||
|
||||
if ($progress_callback) {
|
||||
call_user_func($progress_callback, $this->stats);
|
||||
}
|
||||
}
|
||||
|
||||
// Emit page complete event
|
||||
$this->emit_progress('page_complete', array('processed' => $this->stats['processed']));
|
||||
|
||||
// Check for next page
|
||||
if ($continue && isset($response['@odata.nextLink'])) {
|
||||
// Save progress
|
||||
@@ -138,11 +163,34 @@ class MLS_Sync_Engine {
|
||||
));
|
||||
}
|
||||
|
||||
$start_time = microtime(true);
|
||||
$this->emit_progress('api_request', array(
|
||||
'method' => 'GET',
|
||||
'url' => 'Property (next page)',
|
||||
'params' => array('page' => 'next'),
|
||||
));
|
||||
|
||||
$response = $this->api_client->get_next_page($response['@odata.nextLink']);
|
||||
$elapsed = round((microtime(true) - $start_time) * 1000);
|
||||
|
||||
if (is_wp_error($response)) {
|
||||
$this->emit_progress('api_response', array(
|
||||
'success' => false,
|
||||
'status_code' => 0,
|
||||
'error' => $response->get_error_message(),
|
||||
'record_count' => 0,
|
||||
'response_time' => $elapsed,
|
||||
));
|
||||
throw new Exception($response->get_error_message());
|
||||
}
|
||||
|
||||
$this->emit_progress('api_response', array(
|
||||
'success' => true,
|
||||
'status_code' => 200,
|
||||
'record_count' => isset($response['value']) ? count($response['value']) : 0,
|
||||
'response_time' => $elapsed,
|
||||
'has_more' => isset($response['@odata.nextLink']),
|
||||
));
|
||||
} else {
|
||||
$continue = false;
|
||||
}
|
||||
@@ -211,6 +259,9 @@ class MLS_Sync_Engine {
|
||||
'dry_run' => $dry_run,
|
||||
));
|
||||
|
||||
// Store progress callback for use in process_property
|
||||
$this->progress_callback = $progress_callback;
|
||||
|
||||
if (!$dry_run) {
|
||||
$this->sync_state_id = $this->create_sync_state(self::TYPE_INCREMENTAL);
|
||||
$this->logger->set_sync_state($this->sync_state_id);
|
||||
@@ -226,29 +277,74 @@ class MLS_Sync_Engine {
|
||||
|
||||
try {
|
||||
// Get modified properties (including those marked for deletion)
|
||||
$start_time = microtime(true);
|
||||
$this->emit_progress('api_request', array(
|
||||
'method' => 'GET',
|
||||
'url' => 'Property',
|
||||
'params' => array('type' => 'incremental', 'since' => $last_timestamp),
|
||||
));
|
||||
|
||||
$response = $this->api_client->get_properties_since($last_timestamp, 'Media');
|
||||
$elapsed = round((microtime(true) - $start_time) * 1000);
|
||||
|
||||
if (is_wp_error($response)) {
|
||||
$this->emit_progress('api_response', array(
|
||||
'success' => false,
|
||||
'status_code' => 0,
|
||||
'error' => $response->get_error_message(),
|
||||
'record_count' => 0,
|
||||
'response_time' => $elapsed,
|
||||
));
|
||||
throw new Exception($response->get_error_message());
|
||||
}
|
||||
|
||||
$this->emit_progress('api_response', array(
|
||||
'success' => true,
|
||||
'status_code' => 200,
|
||||
'record_count' => isset($response['value']) ? count($response['value']) : 0,
|
||||
'response_time' => $elapsed,
|
||||
'has_more' => isset($response['@odata.nextLink']),
|
||||
));
|
||||
|
||||
// Process pages
|
||||
while (isset($response['value'])) {
|
||||
foreach ($response['value'] as $property) {
|
||||
$this->process_property($property, $dry_run);
|
||||
|
||||
if ($progress_callback) {
|
||||
call_user_func($progress_callback, $this->stats);
|
||||
}
|
||||
}
|
||||
|
||||
// Emit page complete event
|
||||
$this->emit_progress('page_complete', array('processed' => $this->stats['processed']));
|
||||
|
||||
// Check for next page
|
||||
if (isset($response['@odata.nextLink'])) {
|
||||
$start_time = microtime(true);
|
||||
$this->emit_progress('api_request', array(
|
||||
'method' => 'GET',
|
||||
'url' => 'Property (next page)',
|
||||
'params' => array('page' => 'next'),
|
||||
));
|
||||
|
||||
$response = $this->api_client->get_next_page($response['@odata.nextLink']);
|
||||
$elapsed = round((microtime(true) - $start_time) * 1000);
|
||||
|
||||
if (is_wp_error($response)) {
|
||||
$this->emit_progress('api_response', array(
|
||||
'success' => false,
|
||||
'status_code' => 0,
|
||||
'error' => $response->get_error_message(),
|
||||
'record_count' => 0,
|
||||
'response_time' => $elapsed,
|
||||
));
|
||||
throw new Exception($response->get_error_message());
|
||||
}
|
||||
|
||||
$this->emit_progress('api_response', array(
|
||||
'success' => true,
|
||||
'status_code' => 200,
|
||||
'record_count' => isset($response['value']) ? count($response['value']) : 0,
|
||||
'response_time' => $elapsed,
|
||||
'has_more' => isset($response['@odata.nextLink']),
|
||||
));
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
@@ -324,6 +420,7 @@ class MLS_Sync_Engine {
|
||||
}
|
||||
|
||||
$this->sync_state_id = $sync_state_id;
|
||||
$this->progress_callback = $progress_callback;
|
||||
$this->logger->set_sync_state($sync_state_id);
|
||||
$this->logger->info('Resuming sync', array('sync_state_id' => $sync_state_id));
|
||||
|
||||
@@ -359,12 +456,11 @@ class MLS_Sync_Engine {
|
||||
while (isset($response['value'])) {
|
||||
foreach ($response['value'] as $property) {
|
||||
$this->process_property($property, false);
|
||||
|
||||
if ($progress_callback) {
|
||||
call_user_func($progress_callback, $this->stats);
|
||||
}
|
||||
}
|
||||
|
||||
// Emit page complete event
|
||||
$this->emit_progress('page_complete', array('processed' => $this->stats['processed']));
|
||||
|
||||
if (isset($response['@odata.nextLink'])) {
|
||||
$this->update_sync_state(array(
|
||||
'last_next_link' => $response['@odata.nextLink'],
|
||||
@@ -413,6 +509,11 @@ class MLS_Sync_Engine {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Progress callback reference
|
||||
*/
|
||||
private $progress_callback = null;
|
||||
|
||||
/**
|
||||
* Process a single property record
|
||||
*
|
||||
@@ -439,6 +540,7 @@ class MLS_Sync_Engine {
|
||||
$this->delete_property($listing_key);
|
||||
}
|
||||
$this->stats['deleted']++;
|
||||
$this->emit_progress('property_deleted', array('listing_key' => $listing_key));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -454,8 +556,10 @@ class MLS_Sync_Engine {
|
||||
if ($dry_run) {
|
||||
if ($existing) {
|
||||
$this->stats['updated']++;
|
||||
$this->emit_progress('property_skipped', array('listing_key' => $listing_key));
|
||||
} else {
|
||||
$this->stats['created']++;
|
||||
$this->emit_progress('property_skipped', array('listing_key' => $listing_key));
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -468,17 +572,31 @@ class MLS_Sync_Engine {
|
||||
array('listing_key' => $listing_key)
|
||||
);
|
||||
$this->stats['updated']++;
|
||||
$this->emit_progress('property_updated', array('listing_key' => $listing_key));
|
||||
} else {
|
||||
// Insert new
|
||||
$data['listing_key'] = $listing_key;
|
||||
$data['created_at'] = current_time('mysql');
|
||||
$wpdb->insert($this->db->properties_table(), $data);
|
||||
$this->stats['created']++;
|
||||
$this->emit_progress('property_created', array('listing_key' => $listing_key));
|
||||
}
|
||||
|
||||
// Process media if present
|
||||
if (isset($property['Media']) && is_array($property['Media'])) {
|
||||
$this->media_handler->sync_property_media($listing_key, $property['Media']);
|
||||
$this->media_handler->sync_property_media($listing_key, $property['Media'], false, $this->progress_callback);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit progress event
|
||||
*
|
||||
* @param string $event Event name
|
||||
* @param array $data Event data
|
||||
*/
|
||||
private function emit_progress($event, $data = array()) {
|
||||
if ($this->progress_callback) {
|
||||
call_user_func($this->progress_callback, $event, $data);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user