Add queue-based media download system with rate limiting

- Add download_status, retry_after, queued_at columns to mls_media table
- Add mls_media_log table for download attempt tracking
- Rewrite media handler to queue downloads instead of immediate download
- Add 700ms delay between downloads (25% buffer over 2/sec limit)
- Add 3-hour backoff for rate-limited (429) responses
- Add max 5 attempts before marking as permanently failed
- Add wp mls media command: status, process, reset, logs
- Deprecate wp mls sync media in favor of wp mls media process
- Update documentation with queue system details and cron examples

Media downloads are now separate from property sync:
1. wp mls sync full/incremental - syncs properties, queues media
2. wp mls media process - downloads queued media with rate limiting

Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Hanson.xyz Dev
2025-12-14 22:52:58 -06:00
parent b62867d834
commit 6eadf3d266
5 changed files with 930 additions and 334 deletions
@@ -55,6 +55,13 @@ class MLS_DB {
return $this->get_table_name(MLS_TABLE_SYNC_LOG);
}
/**
* Get media log table name
*/
public function media_log_table() {
return $this->get_table_name(MLS_TABLE_MEDIA_LOG);
}
/**
* Create all database tables
*/
@@ -163,6 +170,9 @@ class MLS_DB {
downloaded_at DATETIME DEFAULT NULL,
download_attempts INT(3) DEFAULT 0,
download_error TEXT DEFAULT NULL,
retry_after DATETIME DEFAULT NULL,
queued_at DATETIME DEFAULT NULL,
download_status VARCHAR(20) DEFAULT 'pending',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
@@ -170,7 +180,10 @@ class MLS_DB {
PRIMARY KEY (id),
UNIQUE KEY listing_media (listing_key, media_key),
KEY listing_key (listing_key),
KEY media_order (media_order)
KEY media_order (media_order),
KEY download_status (download_status),
KEY retry_after (retry_after),
KEY queued_at (queued_at)
) {$charset_collate};";
dbDelta($sql_media);
@@ -239,6 +252,29 @@ class MLS_DB {
) {$charset_collate};";
dbDelta($sql_sync_log);
// Media download log table
$table_media_log = $wpdb->prefix . MLS_TABLE_MEDIA_LOG;
$sql_media_log = "CREATE TABLE {$table_media_log} (
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
media_id BIGINT(20) UNSIGNED NOT NULL,
listing_key VARCHAR(50) NOT NULL,
media_key VARCHAR(100) NOT NULL,
action VARCHAR(30) NOT NULL,
status_code INT(5) DEFAULT NULL,
response_time_ms INT(11) DEFAULT NULL,
error_message TEXT DEFAULT NULL,
url VARCHAR(1000) DEFAULT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id),
KEY media_id (media_id),
KEY listing_key (listing_key),
KEY action (action),
KEY created_at (created_at)
) {$charset_collate};";
dbDelta($sql_media_log);
}
/**
@@ -253,6 +289,7 @@ class MLS_DB {
MLS_TABLE_SYNC_STATE,
MLS_TABLE_RATE_LIMITS,
MLS_TABLE_SYNC_LOG,
MLS_TABLE_MEDIA_LOG,
);
foreach ($tables as $table) {